Skip to content

Z
ZHHT-IRN-BD-ANALYSIS
Commit 5a73990f by 黄准
Options

Merge remote-tracking branch 'origin/master'

parents 811d502b 7d8c33fc
Showing with 1426 additions and 238 deletions
realtime/hologram-streaming/pom.xml
...@@ -60,6 +60,12 @@ ...@@ -60,6 +60,12 @@
</dependency> </dependency>
<dependency> <dependency>
<groupId>org.apache.doris</groupId>
<artifactId>flink-doris-connector-1.14_${scala.binary.version}</artifactId>
<version>1.1.0</version>
</dependency>
<dependency>
<groupId>com.alibaba</groupId> <groupId>com.alibaba</groupId>
<artifactId>fastjson</artifactId> <artifactId>fastjson</artifactId>
<version>1.2.32</version> <version>1.2.32</version>
......
realtime/hologram-streaming/src/main/java/com/zhht/irn/consts/Constant.java
...@@ -12,9 +12,4 @@ public class Constant { ...@@ -12,9 +12,4 @@ public class Constant {
public static final String TRAVEL_TOPIC_NAME = "trips_info"; public static final String TRAVEL_TOPIC_NAME = "trips_info";
public static final String TRAVEL_EVENT_TOPIC_NAME = "trips_event_info"; public static final String TRAVEL_EVENT_TOPIC_NAME = "trips_event_info";
// 周期计算的延迟时间 15秒
public static final int CYCLE_CALCULATE_LATENESS = 15 * 1000;
public static final int INSERT_BATCH_SIZE = 100;
} }
realtime/hologram-streaming/src/main/java/com/zhht/irn/entity/Travel.java
...@@ -116,8 +116,8 @@ public class Travel { ...@@ -116,8 +116,8 @@ public class Travel {
//实际停车次数 //实际停车次数
private int stopCount; private int stopCount;
//停车总时长 //停车总时长, 单位毫秒
private double stopTime; private int stopTime;
//平均车速 //平均车速
private double avgSpeed; private double avgSpeed;
...@@ -392,11 +392,11 @@ public class Travel { ...@@ -392,11 +392,11 @@ public class Travel {
this.stopCount = stopCount; this.stopCount = stopCount;
} }
public double getStopTime() { public int getStopTime() {
return stopTime; return stopTime;
} }
public void setStopTime(double stopTime) { public void setStopTime(int stopTime) {
this.stopTime = stopTime; this.stopTime = stopTime;
} }
......
realtime/hologram-streaming/src/main/java/com/zhht/irn/entity/dto/Location.java
...@@ -2,6 +2,8 @@ package com.zhht.irn.entity.dto; ...@@ -2,6 +2,8 @@ package com.zhht.irn.entity.dto;
import lombok.AllArgsConstructor; import lombok.AllArgsConstructor;
import lombok.Data; import lombok.Data;
import lombok.Getter;
import lombok.NoArgsConstructor;
/** /**
* 位置信息 * 位置信息
...@@ -11,6 +13,7 @@ import lombok.Data; ...@@ -11,6 +13,7 @@ import lombok.Data;
**/ **/
@Data @Data
@AllArgsConstructor @AllArgsConstructor
@NoArgsConstructor
public class Location implements Comparable<Location>{ public class Location implements Comparable<Location>{
private Double longitude ;//可选 经度 private Double longitude ;//可选 经度
......
realtime/hologram-streaming/src/main/java/com/zhht/irn/entity/dto/TravelInfo.java
...@@ -11,7 +11,7 @@ import java.util.List; ...@@ -11,7 +11,7 @@ import java.util.List;
* @create 2022-11-14 13:33{ * @create 2022-11-14 13:33{
**/ **/
@Data @Data
public class TravelInfo { public class TravelInfo implements Comparable<TravelInfo> {
private Integer id ;//必填 ID private Integer id ;//必填 ID
private String crossId ;//必填 路口ID private String crossId ;//必填 路口ID
...@@ -31,6 +31,7 @@ public class TravelInfo { ...@@ -31,6 +31,7 @@ public class TravelInfo {
private Long arrivedTime ;//可选 到达时间，毫秒级别时间戳 private Long arrivedTime ;//可选 到达时间，毫秒级别时间戳
private Double arrivedSpeed ;//可选 到达时速度，单位km/h private Double arrivedSpeed ;//可选 到达时速度，单位km/h
private String inCrossLineId ;//可选 进入路口时车道编号 private String inCrossLineId ;//可选 进入路口时车道编号
private String flowDirection ;//流向
private Long inCrossTime ;//可选 进入路口时间，毫秒级别时间戳 private Long inCrossTime ;//可选 进入路口时间，毫秒级别时间戳
private Double inSpeed ;//可选 进入时速度，单位km/h private Double inSpeed ;//可选 进入时速度，单位km/h
private String outCrossLineId ;//可选 通过路口时车道编号 private String outCrossLineId ;//可选 通过路口时车道编号
...@@ -45,4 +46,9 @@ public class TravelInfo { ...@@ -45,4 +46,9 @@ public class TravelInfo {
private Double crossingTime ;//可选 旅行时长，单位 秒 (s)（驶离时刻-到达时刻） private Double crossingTime ;//可选 旅行时长，单位 秒 (s)（驶离时刻-到达时刻）
private List<Location> locations ;//必填 数组，参考 位置信息字段 private List<Location> locations ;//必填 数组，参考 位置信息字段
private String remark ;//可选 备注 private String remark ;//可选 备注
@Override
public int compareTo(TravelInfo o) {
return (int)(inCrossTime-o.getInCrossTime());
}
} }
realtime/hologram-streaming/src/main/java/com/zhht/irn/entity/metric/DirectionEvalSecondMetric.java
package com.zhht.irn.entity.metric; package com.zhht.irn.entity.metric;
import java.util.Date;
/** /**
* 按方向指标计算结果实体类 * 按方向指标计算结果实体类
* *
...@@ -116,4 +118,6 @@ public class DirectionEvalSecondMetric { ...@@ -116,4 +118,6 @@ public class DirectionEvalSecondMetric {
", accStopCount=" + accStopCount + ", accStopCount=" + accStopCount +
'}'; '}';
} }
} }
realtime/hologram-streaming/src/main/java/com/zhht/irn/functions/CalDirectionSecondMetricFunction.java
...@@ -138,7 +138,6 @@ public class CalDirectionSecondMetricFunction extends KeyedCoProcessFunction<Str ...@@ -138,7 +138,6 @@ public class CalDirectionSecondMetricFunction extends KeyedCoProcessFunction<Str
listState.clear(); listState.clear();
for(Map.Entry<String, Boolean> en : globalRecordStopStatusMap.entrySet()) { for(Map.Entry<String, Boolean> en : globalRecordStopStatusMap.entrySet()) {
if(en.getValue() == true) { if(en.getValue() == true) {
System.out.println("stay stop status car record id:" + en.getKey());
listState.add(en.getKey()); listState.add(en.getKey());
} }
} }
...@@ -146,7 +145,7 @@ public class CalDirectionSecondMetricFunction extends KeyedCoProcessFunction<Str ...@@ -146,7 +145,7 @@ public class CalDirectionSecondMetricFunction extends KeyedCoProcessFunction<Str
if(cachedTravelEvent != null) { if(cachedTravelEvent != null) {
List<TravelEvent> newOne = new ArrayList<>(); List<TravelEvent> newOne = new ArrayList<>();
for(TravelEvent t : cachedTravelEvent) { for(TravelEvent t : cachedTravelEvent) {
// 删除掉这个周期结束时间之间的数据 // 删除掉这个周期结束时间之前的数据
if(Long.parseLong(t.getEventTime()) >= Long.parseLong(cycle.getEndDateTime())) { if(Long.parseLong(t.getEventTime()) >= Long.parseLong(cycle.getEndDateTime())) {
newOne.add(t); newOne.add(t);
} }
......
realtime/hologram-streaming/src/main/java/com/zhht/irn/functions/LaneNormProcessFunction.java
package com.zhht.irn.functions; package com.zhht.irn.functions;
import com.alibaba.druid.pool.DruidPooledConnection; import com.alibaba.druid.pool.DruidPooledConnection;
import com.alibaba.fastjson.JSON;
import com.zhht.irn.entity.*; import com.zhht.irn.entity.*;
import com.zhht.irn.enums.EventType; import com.zhht.irn.enums.EventType;
import com.zhht.irn.job.LaneNormJob;
import com.zhht.irn.utils.*; import com.zhht.irn.utils.*;
import org.apache.flink.api.common.state.*; import org.apache.flink.api.common.state.*;
import org.apache.flink.api.common.time.Time;
import org.apache.flink.api.java.tuple.Tuple2; import org.apache.flink.api.java.tuple.Tuple2;
import org.apache.flink.configuration.Configuration; import org.apache.flink.configuration.Configuration;
import org.apache.flink.streaming.api.functions.KeyedProcessFunction; import org.apache.flink.streaming.api.functions.KeyedProcessFunction;
...@@ -13,11 +16,9 @@ import org.apache.flink.table.api.Table; ...@@ -13,11 +16,9 @@ import org.apache.flink.table.api.Table;
import org.apache.flink.util.Collector; import org.apache.flink.util.Collector;
import java.math.BigDecimal; import java.math.BigDecimal;
import java.sql.Connection; import java.sql.*;
import java.sql.ResultSet;
import java.sql.ResultSetMetaData;
import java.sql.Statement;
import java.util.*; import java.util.*;
import java.util.Date;
import java.util.stream.Collectors; import java.util.stream.Collectors;
import java.util.stream.Stream; import java.util.stream.Stream;
...@@ -39,9 +40,13 @@ public class LaneNormProcessFunction extends KeyedCoProcessFunction<String, Cycl ...@@ -39,9 +40,13 @@ public class LaneNormProcessFunction extends KeyedCoProcessFunction<String, Cycl
// //
private ValueState<Boolean> processCycle; private ValueState<Boolean> processCycle;
private Connection connection; Connection connection;
//每个路口允许的最大时间数据 //每个路口允许的最大时间数据
private static Long maxTravel=50000L; private static Long maxTravel=50000L;
//每天的cycleOrder 不允许重复
private MapState<String,String> dayCycle;
private String loggerLevel="info";
@Override @Override
public void open(Configuration parameters) throws Exception { public void open(Configuration parameters) throws Exception {
...@@ -51,6 +56,16 @@ public class LaneNormProcessFunction extends KeyedCoProcessFunction<String, Cycl ...@@ -51,6 +56,16 @@ public class LaneNormProcessFunction extends KeyedCoProcessFunction<String, Cycl
dict = getRuntimeContext().getMapState(new MapStateDescriptor("dict", String.class, Object.class)); dict = getRuntimeContext().getMapState(new MapStateDescriptor("dict", String.class, Object.class));
processCycle = getRuntimeContext().getState(new ValueStateDescriptor<Boolean>("processCycle", Boolean.class)); processCycle = getRuntimeContext().getState(new ValueStateDescriptor<Boolean>("processCycle", Boolean.class));
connection=DorisUtils.getConnection(); connection=DorisUtils.getConnection();
StateTtlConfig ttlConfig = StateTtlConfig
.newBuilder(Time.hours(3))
.setUpdateType(StateTtlConfig.UpdateType.OnCreateAndWrite)
.setStateVisibility(StateTtlConfig.StateVisibility.NeverReturnExpired)
.build();
MapStateDescriptor<String, String> mapStateDescriptor = new MapStateDescriptor("dayCycle", String.class, String.class);
//
mapStateDescriptor.enableTimeToLive(ttlConfig);
dayCycle = getRuntimeContext().getMapState(mapStateDescriptor);
} }
@Override @Override
...@@ -61,13 +76,37 @@ public class LaneNormProcessFunction extends KeyedCoProcessFunction<String, Cycl ...@@ -61,13 +76,37 @@ public class LaneNormProcessFunction extends KeyedCoProcessFunction<String, Cycl
} }
} }
/**
* 判断jdbc 是否失效 失效重新获取连接
* @param con
* @return
*/
public boolean isDbConnected(Connection con) {
//final String CHECK_SQL_QUERY = "SELECT 1";
try {
if(!con.isClosed() || con!=null){
return true;
}
} catch (SQLException e) {
return false;
}
return false;
}
@Override @Override
public void processElement1(Cycle cycle, KeyedCoProcessFunction<String, Cycle, TravelEvent, List<LaneNorm>>.Context ctx, Collector<List<LaneNorm>> out) throws Exception { public void processElement1(Cycle cycle, KeyedCoProcessFunction<String, Cycle, TravelEvent, List<LaneNorm>>.Context ctx, Collector<List<LaneNorm>> out) throws Exception {
processCycle.update(true); processCycle.update(true);
List<LaneNorm> laneNorms = new ArrayList<>(); List<LaneNorm> laneNorms = new ArrayList<>();
if(!dayCycle.contains(DateUtils.toDateStr(new Date())+"-"+cycle.getCrossCode()+"-"+cycle.getCycleOrder())){
dayCycle.put(DateUtils.toDateStr(new Date())+"-"+cycle.getCrossCode()+"-"+cycle.getCycleOrder(),"true");
}else{
//如果内存中已经存在重复的周期 则不进行计算
return;
}
//初始化字典数据 //初始化字典数据
if (dict.get("phaseDirection") == null || dict.get("laneList") == null || dict.get("dictResult") == null || dict.get("positionList") == null || dict.get("positionNameList") == null) { if (dict.get("phaseDirection") == null || dict.get("laneList") == null || dict.get("dictResult") == null || dict.get("positionList") == null || dict.get("positionNameList") == null) {
if(connection==null){ loggerLevel = applicationUtils.getLoggerLevel();
if(isDbConnected(connection)){
connection=DorisUtils.getConnection(); connection=DorisUtils.getConnection();
} }
//查询相位方向信息 //查询相位方向信息
...@@ -128,17 +167,17 @@ public class LaneNormProcessFunction extends KeyedCoProcessFunction<String, Cycl ...@@ -128,17 +167,17 @@ public class LaneNormProcessFunction extends KeyedCoProcessFunction<String, Cycl
List<Map> positionNameList = dict.get("positionNameList").stream().filter(recode -> f1.getCrossCode().equals(recode.get("cross_id"))).collect(Collectors.toList()); List<Map> positionNameList = dict.get("positionNameList").stream().filter(recode -> f1.getCrossCode().equals(recode.get("cross_id"))).collect(Collectors.toList());
for (Map map : dictResult) { for (Map map : dictResult) {
//根据车道字典计算车道指标 //根据车道字典计算车道指标
LaneNorm laneNorm = getLaneNorm(map, f1, carForLane, laneList); LaneNorm laneNorm = getLaneNorm(map, f1, carForLane, laneList,loggerLevel);
list.add(laneNorm); list.add(laneNorm);
//交通流量为0得不参与计算 //交通流量为0得不参与计算
if (laneNorm != null && laneNorm.getTrafficCapacity() != 0d) { if (laneNorm != null) {
laneNorms.add(laneNorm); laneNorms.add(laneNorm);
} }
} }
//获取所有方向的指标 //获取所有方向的指标
getPositionNorm(list, positionList, cycle, laneNorms); getPositionNorm(list, positionList, f1, laneNorms);
//获取所有流向的指标 //获取所有流向的指标
getPositionNameNorm(list, positionNameList, cycle, laneNorms); getPositionNameNorm(list, positionNameList, f1, laneNorms,loggerLevel);
if(laneNorms.size()>0){ if(laneNorms.size()>0){
out.collect(laneNorms); out.collect(laneNorms);
} }
...@@ -148,7 +187,6 @@ public class LaneNormProcessFunction extends KeyedCoProcessFunction<String, Cycl ...@@ -148,7 +187,6 @@ public class LaneNormProcessFunction extends KeyedCoProcessFunction<String, Cycl
List<TravelEvent> travelEvents1 = cycleTravelTemp.get(cycle.getCrossCode())==null?new ArrayList<>():cycleTravelTemp.get(cycle.getCrossCode()); List<TravelEvent> travelEvents1 = cycleTravelTemp.get(cycle.getCrossCode())==null?new ArrayList<>():cycleTravelTemp.get(cycle.getCrossCode());
//未参与计算的事件数据 //未参与计算的事件数据
List<TravelEvent> futureEvents = travelEvents.stream().filter(a -> Long.parseLong(a.getEventTime()) > Long.parseLong(cycle.getEndDateTime())).collect(Collectors.toList()); List<TravelEvent> futureEvents = travelEvents.stream().filter(a -> Long.parseLong(a.getEventTime()) > Long.parseLong(cycle.getEndDateTime())).collect(Collectors.toList());
//计算中进入的数据 未参与计算的事件数据 本次计算中未发生进入事件的数据进行合并 放入内存中 下周期继续计算 //计算中进入的数据 未参与计算的事件数据 本次计算中未发生进入事件的数据进行合并 放入内存中 下周期继续计算
collect.addAll(travelEvents1); collect.addAll(travelEvents1);
collect.addAll(futureEvents); collect.addAll(futureEvents);
...@@ -181,7 +219,6 @@ public class LaneNormProcessFunction extends KeyedCoProcessFunction<String, Cycl ...@@ -181,7 +219,6 @@ public class LaneNormProcessFunction extends KeyedCoProcessFunction<String, Cycl
} }
} }
@Override @Override
public void onTimer(long timestamp, OnTimerContext ctx, Collector<List<LaneNorm>> out) throws Exception { public void onTimer(long timestamp, OnTimerContext ctx, Collector<List<LaneNorm>> out) throws Exception {
super.onTimer(timestamp, ctx, out); super.onTimer(timestamp, ctx, out);
...@@ -208,14 +245,24 @@ public class LaneNormProcessFunction extends KeyedCoProcessFunction<String, Cycl ...@@ -208,14 +245,24 @@ public class LaneNormProcessFunction extends KeyedCoProcessFunction<String, Cycl
} }
return AllGreenTime + ""; return AllGreenTime + "";
} }
/**
private static String gettEffectGreenTimeForDirction(List<Stage> stageList, String position,String direction) { * 北 直行
* @param cycle
* @param position
* @param direction
* @return
*/
private static String gettEffectGreenTimeForDirction(Cycle cycle, String position,String direction,String loggerLevel) {
List<Stage> stageList = cycle.getStageList();
int AllGreenTime = 0; int AllGreenTime = 0;
for (Stage stage : stageList) { for (Stage stage : stageList) {
List<StageForDirection> stageForDirectionList = stage.getStageForDirectionList(); List<StageForDirection> stageForDirectionList = stage.getStageForDirectionList();
for (StageForDirection dir : stageForDirectionList) { for (StageForDirection dir : stageForDirectionList) {
if (position.equals(dir.getDirection()) && direction.equals(dir.getTraffic_flow_direction())) { if (position.equals(dir.getDirection()) && direction.equals(dir.getTraffic_flow_direction())) {
AllGreenTime += stage.getValidGreen(); AllGreenTime += stage.getValidGreen();
if("debug".equals(loggerLevel)) {
System.out.println("cycleorder->"+cycle.getCycleOrder()+" 路口id->"+cycle.getCrossCode()+" "+position + direction+"命中->相位"+stage.getPhaseValue() + " 相位有效绿灯时间为->"+stage.getValidGreen() );
}
//同一个相位累加一次即可 //同一个相位累加一次即可
break; break;
} }
...@@ -223,7 +270,6 @@ public class LaneNormProcessFunction extends KeyedCoProcessFunction<String, Cycl ...@@ -223,7 +270,6 @@ public class LaneNormProcessFunction extends KeyedCoProcessFunction<String, Cycl
} }
return AllGreenTime + ""; return AllGreenTime + "";
} }
/** /**
* 提前计算所有相位允许的流向 * 提前计算所有相位允许的流向
* *
...@@ -244,8 +290,6 @@ public class LaneNormProcessFunction extends KeyedCoProcessFunction<String, Cycl ...@@ -244,8 +290,6 @@ public class LaneNormProcessFunction extends KeyedCoProcessFunction<String, Cycl
} }
return stageForDirections; return stageForDirections;
} }
/** /**
* 生成流向指标 * 生成流向指标
* *
...@@ -254,7 +298,7 @@ public class LaneNormProcessFunction extends KeyedCoProcessFunction<String, Cycl ...@@ -254,7 +298,7 @@ public class LaneNormProcessFunction extends KeyedCoProcessFunction<String, Cycl
* @param cycle * @param cycle
* @param collectList * @param collectList
*/ */
private static void getPositionNameNorm(List<LaneNorm> list, List<Map> positionNameList, Cycle cycle, List<LaneNorm> collectList) { private static void getPositionNameNorm(List<LaneNorm> list, List<Map> positionNameList, Cycle cycle, List<LaneNorm> collectList,String loggerLevel) {
if (list.size() > 0) { if (list.size() > 0) {
for (Map map : positionNameList) { for (Map map : positionNameList) {
String position = String.valueOf(map.get("position")); String position = String.valueOf(map.get("position"));
...@@ -268,8 +312,8 @@ public class LaneNormProcessFunction extends KeyedCoProcessFunction<String, Cycl ...@@ -268,8 +312,8 @@ public class LaneNormProcessFunction extends KeyedCoProcessFunction<String, Cycl
laneNorm.setType("1"); laneNorm.setType("1");
laneNorm.setCycleStartTime(DateUtils.stampToTime(cycle.getBeginDateTime())); laneNorm.setCycleStartTime(DateUtils.stampToTime(cycle.getBeginDateTime()));
laneNorm.setCycleEndTime(DateUtils.stampToTime(cycle.getEndDateTime())); laneNorm.setCycleEndTime(DateUtils.stampToTime(cycle.getEndDateTime()));
//获取所有这个方向得车道 交通流量为0得不参与计算 //获取所有这个方向得车道
List<LaneNorm> collect = list.stream().filter(recode -> position.equals(recode.getDirection()) && recode.getLaneName().indexOf(name) != -1 && recode.getTrafficCapacity() != 0d).collect(Collectors.toList()); List<LaneNorm> collect = list.stream().filter(recode -> position.equals(recode.getDirection())&&recode.getLaneName().indexOf(name) != -1).collect(Collectors.toList());
if (collect.size() == 0) { if (collect.size() == 0) {
continue; continue;
} }
...@@ -282,10 +326,12 @@ public class LaneNormProcessFunction extends KeyedCoProcessFunction<String, Cycl ...@@ -282,10 +326,12 @@ public class LaneNormProcessFunction extends KeyedCoProcessFunction<String, Cycl
//交通流量求和 //交通流量求和
laneNorm.setTrafficCapacity(collect.stream().mapToDouble(LaneNorm::getTrafficCapacity).sum()); laneNorm.setTrafficCapacity(collect.stream().mapToDouble(LaneNorm::getTrafficCapacity).sum());
//剩余承载力求和 //剩余承载力求和
double sum2 = collect.stream().mapToDouble(LaneNorm::getResidualCapacity).sum(); double sum2 = collect.stream().reduce(0.0,(x,y)->x+(y.getPassCapacity()-y.getTrafficCapacity()),Double::sum);
laneNorm.setResidualCapacity(new BigDecimal(sum2).setScale(5, BigDecimal.ROUND_HALF_UP).doubleValue()); //double sum2 = collect.stream().mapToDouble(LaneNorm::getResidualCapacity).sum();
//有效绿灯时间 相位下此方向非右转的绿灯有效时间之和 laneNorm.setResidualCapacity(new BigDecimal(sum2<=0?0d:sum2).setScale(5, BigDecimal.ROUND_HALF_UP).doubleValue());
laneNorm.setEffectGreenTime(gettEffectGreenTimeForDirction(cycle.getStageList(), position,name)); //laneNorm.setResidualCapacity(new BigDecimal(sum2).setScale(5, BigDecimal.ROUND_HALF_UP).doubleValue());
//有效绿灯时间 相位下此方向此流向有效绿灯时间之和
laneNorm.setEffectGreenTime(gettEffectGreenTimeForDirction(cycle, position,name,loggerLevel));
//识别空间空间占有率 车道长度需要交通流量为0得数据 //识别空间空间占有率 车道长度需要交通流量为0得数据
int sum = list.stream().filter(recode -> position.equals(recode.getDirection()) && recode.getLaneName().indexOf(name) != -1).mapToInt(LaneNorm::getLaneLength).sum(); int sum = list.stream().filter(recode -> position.equals(recode.getDirection()) && recode.getLaneName().indexOf(name) != -1).mapToInt(LaneNorm::getLaneLength).sum();
int sum1 = collect.stream().mapToInt(LaneNorm::getMaxQueueLength).sum(); int sum1 = collect.stream().mapToInt(LaneNorm::getMaxQueueLength).sum();
...@@ -298,7 +344,6 @@ public class LaneNormProcessFunction extends KeyedCoProcessFunction<String, Cycl ...@@ -298,7 +344,6 @@ public class LaneNormProcessFunction extends KeyedCoProcessFunction<String, Cycl
} }
} }
} }
/** /**
* 生成方向指标 * 生成方向指标
* *
...@@ -320,8 +365,8 @@ public class LaneNormProcessFunction extends KeyedCoProcessFunction<String, Cycl ...@@ -320,8 +365,8 @@ public class LaneNormProcessFunction extends KeyedCoProcessFunction<String, Cycl
laneNorm.setType("0"); laneNorm.setType("0");
laneNorm.setCycleStartTime(DateUtils.stampToTime(cycle.getBeginDateTime())); laneNorm.setCycleStartTime(DateUtils.stampToTime(cycle.getBeginDateTime()));
laneNorm.setCycleEndTime(DateUtils.stampToTime(cycle.getEndDateTime())); laneNorm.setCycleEndTime(DateUtils.stampToTime(cycle.getEndDateTime()));
//获取所有这个方向得车道 交通流量为0得不参与计算 //获取所有这个方向得车道
List<LaneNorm> collect = list.stream().filter(recode -> position.equals(recode.getDirection()) && recode.getTrafficCapacity() != 0d).collect(Collectors.toList()); List<LaneNorm> collect = list.stream().filter(recode -> position.equals(recode.getDirection())).collect(Collectors.toList());
if (collect.size() == 0) { if (collect.size() == 0) {
continue; continue;
} }
...@@ -333,8 +378,9 @@ public class LaneNormProcessFunction extends KeyedCoProcessFunction<String, Cycl ...@@ -333,8 +378,9 @@ public class LaneNormProcessFunction extends KeyedCoProcessFunction<String, Cycl
//交通流量求和 //交通流量求和
laneNorm.setTrafficCapacity(collect.stream().mapToDouble(LaneNorm::getTrafficCapacity).sum()); laneNorm.setTrafficCapacity(collect.stream().mapToDouble(LaneNorm::getTrafficCapacity).sum());
//剩余承载力求和 //剩余承载力求和
double sum2 = collect.stream().mapToDouble(LaneNorm::getResidualCapacity).sum(); double sum2 = collect.stream().reduce(0.0,(x,y)->x+(y.getPassCapacity()-y.getTrafficCapacity()),Double::sum);
laneNorm.setResidualCapacity(new BigDecimal(sum2).setScale(5, BigDecimal.ROUND_HALF_UP).doubleValue()); //double sum2 = collect.stream().mapToDouble(LaneNorm::getResidualCapacity).sum();
laneNorm.setResidualCapacity(new BigDecimal(sum2<=0?0d:sum2).setScale(5, BigDecimal.ROUND_HALF_UP).doubleValue());
//有效绿灯时间 相位下此方向非右转得绿灯有效时间之和 //有效绿灯时间 相位下此方向非右转得绿灯有效时间之和
laneNorm.setEffectGreenTime(gettEffectGreenTimeForPosition(cycle.getStageList(), position)); laneNorm.setEffectGreenTime(gettEffectGreenTimeForPosition(cycle.getStageList(), position));
//识别空间空间占有率 车道长度需要交通流量为0得数据 //识别空间空间占有率 车道长度需要交通流量为0得数据
...@@ -350,6 +396,109 @@ public class LaneNormProcessFunction extends KeyedCoProcessFunction<String, Cycl ...@@ -350,6 +396,109 @@ public class LaneNormProcessFunction extends KeyedCoProcessFunction<String, Cycl
} }
} }
public static void main(String[] args) {
try {
Connection connection = getConnection();
List<Map> dictResult = getDictResult(connection, "SELECT\n" +
" a.ID id,\n" +
" a.Direction direction,\n" +
" b.Name traffic_flow_direction,\n" +
" c.PID phase_no,\n" +
" c.PName phase_name\n" +
"FROM dim_phase_movement a\n" +
"LEFT JOIN dim_traffic_flow_direction b\n" +
" on a.TrafficFlowId = b.ID\n" +
"LEFT JOIN dim_dic_phase c\n" +
" on a.PhaseId = c.ID");
connection.close();
Cycle cycle = setCycle();
Cycle cycle1 = cycle;
for (Stage stage : cycle1.getStageList()) {
stage.setStageForDirectionList(getStageForDiretion(stage, dictResult));
}
String s = gettEffectGreenTimeForDirction(cycle, "北", "直行","debug");
//System.out.println(s);
}catch (Exception e){
System.out.println(e);
}
}
public static Cycle setCycle()throws Exception{
String cycleJson="{\n" +
" \"id\": 39240,\n" +
" \"crossCode\": \"13070200022\",\n" +
" \"signalCode\": \"22\",\n" +
" \"beginControlModeCode\": \"21\",\n" +
" \"endControlModeCode\": \"21\",\n" +
" \"beginPlanId\": 33638,\n" +
" \"endPlanId\": 0,\n" +
" \"beginDateTime\": 1673084326000,\n" +
" \"endDateTime\": 1673084481000,\n" +
" \"duration\": 155,\n" +
" \"cycleOrder\": 2695,\n" +
" \"stageList\": [{\n" +
" \"id\": 97300,\n" +
" \"phaseValue\": \"5\",\n" +
" \"beginDateTime\": 1673084326000,\n" +
" \"endDateTime\": 1673084370000,\n" +
" \"beginControlModeCode\": \"21\",\n" +
" \"endControlModeCode\": \"21\",\n" +
" \"beginPlanId\": 33638,\n" +
" \"endPlanId\": 0,\n" +
" \"duration\": 44,\n" +
" \"green\": 41,\n" +
" \"yellow\": 3,\n" +
" \"allRed\": 0,\n" +
" \"phaseOrder\": 1\n" +
" }, {\n" +
" \"id\": 97302,\n" +
" \"phaseValue\": \"6\",\n" +
" \"beginDateTime\": 1673084370000,\n" +
" \"endDateTime\": 1673084417000,\n" +
" \"beginControlModeCode\": \"21\",\n" +
" \"endControlModeCode\": \"21\",\n" +
" \"beginPlanId\": 33638,\n" +
" \"endPlanId\": 0,\n" +
" \"duration\": 46,\n" +
" \"green\": 43,\n" +
" \"yellow\": 3,\n" +
" \"allRed\": 0,\n" +
" \"phaseOrder\": 2\n" +
" }, {\n" +
" \"id\": 97303,\n" +
" \"phaseValue\": \"8\",\n" +
" \"beginDateTime\": 1673084417000,\n" +
" \"endDateTime\": 1673084455000,\n" +
" \"beginControlModeCode\": \"21\",\n" +
" \"endControlModeCode\": \"21\",\n" +
" \"beginPlanId\": 33638,\n" +
" \"endPlanId\": 0,\n" +
" \"duration\": 39,\n" +
" \"green\": 36,\n" +
" \"yellow\": 3,\n" +
" \"allRed\": 0,\n" +
" \"phaseOrder\": 3\n" +
" }, {\n" +
" \"id\": 97305,\n" +
" \"phaseValue\": \"0\",\n" +
" \"beginDateTime\": 1673084455000,\n" +
" \"endDateTime\": 1673084481000,\n" +
" \"beginControlModeCode\": \"21\",\n" +
" \"endControlModeCode\": \"21\",\n" +
" \"beginPlanId\": 33638,\n" +
" \"endPlanId\": 0,\n" +
" \"duration\": 26,\n" +
" \"green\": 23,\n" +
" \"yellow\": 3,\n" +
" \"allRed\": 0,\n" +
" \"phaseOrder\": 4\n" +
" }]\n" +
"}";
Cycle cycle = JSON.parseObject(cycleJson, Cycle.class);
Cycle cycle1 = LaneNormJob.getCycle(cycle);
return cycle1;
}
/** /**
* @param map 车道信息 * @param map 车道信息
* @param cycle 周期信息 * @param cycle 周期信息
...@@ -357,7 +506,7 @@ public class LaneNormProcessFunction extends KeyedCoProcessFunction<String, Cycl ...@@ -357,7 +506,7 @@ public class LaneNormProcessFunction extends KeyedCoProcessFunction<String, Cycl
* @param laneList 车道相关信息 * @param laneList 车道相关信息
* @return * @return
*/ */
private static LaneNorm getLaneNorm(Map map, Cycle cycle, Map<String, List<TravelEvent>> car, List<Map> laneList) { private static LaneNorm getLaneNorm(Map map, Cycle cycle, Map<String, List<TravelEvent>> car, List<Map> laneList,String loggerLevel) {
LaneNorm laneNorm = new LaneNorm(); LaneNorm laneNorm = new LaneNorm();
//车道属性 分别是车道编码 方向 //车道属性 分别是车道编码 方向
String lane_id = String.valueOf(map.get("lane_id")); String lane_id = String.valueOf(map.get("lane_id"));
...@@ -378,10 +527,11 @@ public class LaneNormProcessFunction extends KeyedCoProcessFunction<String, Cycl ...@@ -378,10 +527,11 @@ public class LaneNormProcessFunction extends KeyedCoProcessFunction<String, Cycl
laneNorm.setType("2"); laneNorm.setType("2");
laneNorm.setCycleStartTime(DateUtils.stampToTime(cycle.getBeginDateTime())); laneNorm.setCycleStartTime(DateUtils.stampToTime(cycle.getBeginDateTime()));
laneNorm.setCycleEndTime(DateUtils.stampToTime(cycle.getEndDateTime())); laneNorm.setCycleEndTime(DateUtils.stampToTime(cycle.getEndDateTime()));
//如果交通流量为0，则不统计此车道指标数据 /*//如果交通流量为0，则不统计此车道指标数据
//2.0版本 参与计算
if (trafficCapacity == 0d) { if (trafficCapacity == 0d) {
return laneNorm; return laneNorm;
} }*/
//车道饱和度 //车道饱和度
int saturation_flow_rate = getSaturation(map); int saturation_flow_rate = getSaturation(map);
//车道有效绿灯时间 //车道有效绿灯时间
...@@ -417,6 +567,9 @@ public class LaneNormProcessFunction extends KeyedCoProcessFunction<String, Cycl ...@@ -417,6 +567,9 @@ public class LaneNormProcessFunction extends KeyedCoProcessFunction<String, Cycl
laneNorm.setSpotSpaceShare(d1 > 1d ? 1d : d1); laneNorm.setSpotSpaceShare(d1 > 1d ? 1d : d1);
//计算空间占有长度 //计算空间占有长度
laneNorm.setSpotSpaceLength(maxQueueLength); laneNorm.setSpotSpaceLength(maxQueueLength);
if("debug".equals(loggerLevel)){
//System.out.println("车道指标信息->"+laneNorm.toString()+" 车道饱和度->"+saturation_flow_rate+" 有效绿灯时间为->"+effcetGreenTime[1]+" 周期有效绿灯时间->"+cycle.getDuration());
}
return laneNorm; return laneNorm;
} }
...@@ -478,6 +631,8 @@ public class LaneNormProcessFunction extends KeyedCoProcessFunction<String, Cycl ...@@ -478,6 +631,8 @@ public class LaneNormProcessFunction extends KeyedCoProcessFunction<String, Cycl
int AllGreenTime = 0; int AllGreenTime = 0;
//从字典中获取当前车道所有流向信息 //从字典中获取当前车道所有流向信息
List<Map> currentLane = laneList.stream().filter(a -> laneId.equals(String.valueOf(a.get("lane_id")))).collect(Collectors.toList()); List<Map> currentLane = laneList.stream().filter(a -> laneId.equals(String.valueOf(a.get("lane_id")))).collect(Collectors.toList());
//已经参与计算的相位，不累计计算
Map<Object, Object> HasStage = new HashMap<>();
//获取周期的相位信息 //获取周期的相位信息
for (Map map : currentLane) { for (Map map : currentLane) {
int greeTime = 0; int greeTime = 0;
...@@ -487,8 +642,12 @@ public class LaneNormProcessFunction extends KeyedCoProcessFunction<String, Cycl ...@@ -487,8 +642,12 @@ public class LaneNormProcessFunction extends KeyedCoProcessFunction<String, Cycl
List<StageForDirection> stageForDirectionList = stage.getStageForDirectionList(); List<StageForDirection> stageForDirectionList = stage.getStageForDirectionList();
for (StageForDirection dir : stageForDirectionList) { for (StageForDirection dir : stageForDirectionList) {
if (position.equals(dir.getDirection()) && name.equals(dir.getTraffic_flow_direction())) { if (position.equals(dir.getDirection()) && name.equals(dir.getTraffic_flow_direction())) {
//已经参与计算的相位，不参与累积绿灯时间计算
if(!HasStage.containsKey(stage.getPhaseValue())){
AllGreenTime += stage.getValidGreen();
HasStage.put(stage.getPhaseValue(),"1");
}
greeTime += stage.getValidGreen(); greeTime += stage.getValidGreen();
AllGreenTime += stage.getValidGreen();
} }
} }
} }
...@@ -522,7 +681,7 @@ public class LaneNormProcessFunction extends KeyedCoProcessFunction<String, Cycl ...@@ -522,7 +681,7 @@ public class LaneNormProcessFunction extends KeyedCoProcessFunction<String, Cycl
maxQueneLength=queneLength; maxQueneLength=queneLength;
} }
} }
//旧版逻辑 根据有到达时间 没有进入时间的车辆取 最大排队长度 //todo 旧版逻辑根据有到达时间 没有进入时间的车辆取 最大排队长度
/*i = laneCarMap.get(laneId).stream(). /*i = laneCarMap.get(laneId).stream().
filter(a -> StringUtils.isNotEmpty(a.getArrayTime()) && filter(a -> StringUtils.isNotEmpty(a.getArrayTime()) &&
StringUtils.isEmpty(a.getInCorssTime())).collect(Collectors.toList()).size();*/ StringUtils.isEmpty(a.getInCorssTime())).collect(Collectors.toList()).size();*/
...@@ -533,12 +692,10 @@ public class LaneNormProcessFunction extends KeyedCoProcessFunction<String, Cycl ...@@ -533,12 +692,10 @@ public class LaneNormProcessFunction extends KeyedCoProcessFunction<String, Cycl
return (maxQueneLength * 6); return (maxQueneLength * 6);
} }
private static DruidPooledConnection getConnection() throws Exception { private static Connection getConnection() throws Exception {
//修改为从连接池中取连接 //修改为从连接池中取连接
//DruidPooledConnection connection = DruidConnectPoolUtils.getConnection(); Connection connection = DorisUtils.getConnection();
//Connection connection = DorisUtils.getConnection(); return connection;
//return connection;
return null;
} }
private static List<Map> getDictResult(Connection connection, String sql) throws Exception { private static List<Map> getDictResult(Connection connection, String sql) throws Exception {
......
realtime/hologram-streaming/src/main/java/com/zhht/irn/functions/TravelCarInfoCoProcessFunction.java 0 → 100644
package com.zhht.irn.functions;
import com.alibaba.druid.pool.DruidDataSource;
import com.alibaba.druid.pool.DruidPooledConnection;
import com.zhht.irn.entity.dto.*;
import com.zhht.irn.utils.DorisUtils;
import com.zhht.irn.utils.DruidConnectPoolUtils;
import org.apache.flink.api.common.state.*;
import org.apache.flink.api.common.time.Time;
import org.apache.flink.api.common.typeinfo.TypeInformation;
import org.apache.flink.configuration.Configuration;
import org.apache.flink.streaming.api.functions.co.CoProcessFunction;
import org.apache.flink.util.Collector;
import org.apache.log4j.Logger;
import java.sql.Connection;
import java.sql.PreparedStatement;
import java.sql.ResultSet;
import java.sql.SQLException;
import java.text.SimpleDateFormat;
import java.util.*;
/**
* 行车情况数据处理函数，双流connect后处理
*
* @author ruimeng
* @create 2022-11-14 17:16
**/
public class TravelCarInfoCoProcessFunction extends CoProcessFunction<TravelInfo, CycleSignalData, List<TravelLineSinkInfo>> {
private static Logger logger = Logger.getLogger(TravelCarInfoCoProcessFunction.class);
//定义状态，用于存储旅行信息，当周期信号数据过来的时候，触发计算
//状态数据存储结构为 Map（路口id--》（Map（车道id-》（List（旅行信息））））） 一辆车只有一条旅行信息
// 这样就把每个路口的数据进行分流，同时，对每个路口的数据，也按照了不同的车道进行了分流，同一个路口，同一个车道的数据存在一个list中
private MapState<String, Map<String, List<TravelInfo>>> travelInfoState;
private MapState<Integer, CycleSignalData> cycleSignalDataState; //缓存周期数据（避免周期数据先到，而旅行数数据后到的情况）
private MapState<Integer, CycleSignalData> alreadyHandleCycleSignalDataState; //已经处理过的周期数据，避免重复触发
private ListState<Long> cycleSectionState ; //周期数据切片 ，记录每个周期的开始时间，以便于进行对locations切分到不同周期，计算停车次数
private static Map<String, Map<String, String>> dim_cnt_cross_lane_position;
SimpleDateFormat sdf = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss");
Connection connection;
@Override
public void open(Configuration parameters) throws Exception {
super.open(parameters);
MapStateDescriptor<String, Map<String, List<TravelInfo>>> travelInfoStateDescriptor =
new MapStateDescriptor("travelInfoState",
TypeInformation.of(String.class), TypeInformation.of(Map.class));
MapStateDescriptor<Integer, CycleSignalData> cycleSignalDataStateDescriptor =
new MapStateDescriptor("cycleSignalDataState",
TypeInformation.of(Integer.class), TypeInformation.of(CycleSignalData.class));
MapStateDescriptor<Integer, CycleSignalData> alreadyHandleCycleSignalDataStateDescriptor =
new MapStateDescriptor("alreadyHandleCycleSignalDataState",
TypeInformation.of(Integer.class), TypeInformation.of(CycleSignalData.class));
ListStateDescriptor cycleSectionStateDescriptor = new ListStateDescriptor("cycleSectionState", TypeInformation.of(Long.class));
//待计算的周期数据
StateTtlConfig stateTtlConfig4CycleData = StateTtlConfig
// 状态有效时间 3小时
.newBuilder(Time.hours(3))
// 设置状态的更新类型
.setUpdateType(StateTtlConfig.UpdateType.OnCreateAndWrite)
// 已过期还未被清理掉的状态数据不返回给用户
.setStateVisibility(StateTtlConfig.StateVisibility.NeverReturnExpired)
// 过期对象的清理策略 全量清理
.cleanupFullSnapshot()
.build();
cycleSignalDataStateDescriptor.enableTimeToLive(stateTtlConfig4CycleData);
alreadyHandleCycleSignalDataStateDescriptor.enableTimeToLive(stateTtlConfig4CycleData);
travelInfoState = getRuntimeContext().getMapState(travelInfoStateDescriptor);
cycleSignalDataState = getRuntimeContext().getMapState(cycleSignalDataStateDescriptor);
alreadyHandleCycleSignalDataState = getRuntimeContext().getMapState(alreadyHandleCycleSignalDataStateDescriptor);
cycleSectionState = getRuntimeContext().getListState(cycleSectionStateDescriptor);
dim_cnt_cross_lane_position = getLaneDimData();
connection = DorisUtils.getConnection();
}
@Override
public void close() throws Exception {
super.close();
if(connection!=null){
connection.close();
}
}
//把每个路口的数据 车辆旅行信息数据，存储在状态中，当周期数据进来的时候，再进行触发计算
// 所有旅行信息存储在状态中，当周期数据过来的时候，去状态中挑选属于这个周期内的旅行信息的数据即可
@Override
public void processElement1(TravelInfo value, CoProcessFunction<TravelInfo, CycleSignalData, List<TravelLineSinkInfo>>.Context ctx,
Collector<List<TravelLineSinkInfo>> out) throws Exception {
if(value.getInCrossTime() < 0 || value.getArrivedTime() < 0){
logger.info("路口id"+value.getCrossId() +"车辆id"+value.getCarId() + "进入路口时间："+sdf.format(value.getInCrossTime())+"数据存在异常-作废");
return ;
}
// System.out.println("路口id"+value.getCrossId() +"车辆id"+value.getCarId() + "进入路口时间："+sdf.format(value.getInCrossTime()));
logger.info("路口id"+value.getCrossId() +"车辆id"+value.getCarId() +
"进入路口时间："+sdf.format(value.getInCrossTime())+"到达时间"+sdf.format(value.getArrivedTime()));
String crossId = value.getCrossId();
Iterator<Map<String, List<TravelInfo>>> iterator = travelInfoState.values().iterator();
//如果状态为空，则进行初始化
if (!iterator.hasNext()) {
Map<String, Map<String, List<TravelInfo>>> initMap = new HashMap<>();
travelInfoState.putAll(initMap);
}
//获取当前路口的数据
Map<String, List<TravelInfo>> crossListMap = travelInfoState.get(crossId);
//当前路口没有数据
if(crossListMap == null ){
Map<String, List<TravelInfo>> crossMap = new HashMap<>();
List<TravelInfo> list = new ArrayList<>();
list.add(value);
//车道编号，当前车道的旅行信息
crossMap.put(value.getInCrossLineId(), list);
travelInfoState.put(crossId, crossMap);
}
//当前路口有数据，但是当前车道没有数据
if (crossListMap != null && !crossListMap.containsKey(value.getInCrossLineId())) {
List<TravelInfo> list = new ArrayList<>();
list.add(value);
//车道编号，当前车道的旅行信息
crossListMap.put(value.getInCrossLineId(), list);
travelInfoState.put(crossId, crossListMap);
//当前路口有数据，且 该路口的当前车道也有数据进来了,则把这条旅行信息，放入对应路口下的对应的车道中去
}
//当前路口有数据，而且 当前车道 也有数据
if(crossListMap != null && crossListMap.containsKey(value.getInCrossLineId())) {
List<TravelInfo> lineTravelInfos = crossListMap.get(value.getInCrossLineId());
//如果，当前车道的数据，积压超过5000条，则进行清理操作
if(lineTravelInfos.size()>5000){
logger.info("移除积压的数据。。。。");
List<TravelInfo> travelInfoList = orderByInCrossTime(lineTravelInfos);
List<TravelInfo> travelInfoListNew = travelInfoList.subList(500, travelInfoList.size());
travelInfoListNew.add(value);
crossListMap.put(value.getInCrossLineId(), travelInfoListNew);
travelInfoState.put(crossId, crossListMap);
} else {
lineTravelInfos.add(value);
crossListMap.put(value.getInCrossLineId(), lineTravelInfos);
travelInfoState.put(crossId, crossListMap);
}
}
}
//来一条周期数据，触发这个周期内的旅行数据
@Override
public void processElement2(CycleSignalData value, CoProcessFunction<TravelInfo, CycleSignalData, List<TravelLineSinkInfo>>.Context ctx,
Collector<List<TravelLineSinkInfo>> out) {
try {
//System.out.println("路口id"+value.getCrossCode() +"周期id"+value.getCycleOrder() + "周期时间："+sdf.format(value.getBeginDateTime())+"~"+sdf.format(value.getEndDateTime()));
logger.info("路口id"+value.getCrossCode() +"周期id"+value.getCycleOrder() + "周期时间："+sdf.format(value.getBeginDateTime())+"~"+sdf.format(value.getEndDateTime()));
//判断当前周期是否执行过了,存在周期数据重复下发的情况，只触发一次计算
if (alreadyHandleCycleSignalDataState.contains(value.getCycleOrder())) {
logger.info("周期数据重复----当前路口id是" + value.getCrossCode() + "当前周期已经计算完成了" + value.getCycleOrder());
return;
}
//记录周期数据时间切片，以便划分locations
cycleSectionState.add(value.getBeginDateTime());
String crossCode = value.getCrossCode();
// System.out.println("当前路口是---->"+crossCode);
Integer cycleOrder = value.getCycleOrder();
//周期开始时间，周期结束时间
Long beginDateTime = value.getBeginDateTime();
Long endDateTime = value.getEndDateTime();
//如果，存在之前周期没有计算的，先计算之前的周期的数据
if (!cycleSignalDataState.isEmpty()) {
Iterator<CycleSignalData> iterator = cycleSignalDataState.values().iterator();
List<Integer> listToRemove = new ArrayList<>();
while (iterator.hasNext()) {
CycleSignalData beforeCycle = iterator.next();
Map<String, List<TravelInfo>> beforeCycleDataListMap = filterThisCycleData(crossCode, beforeCycle.getBeginDateTime(), beforeCycle.getEndDateTime());
if (beforeCycleDataListMap != null) {
List<TravelLineSinkInfo> travelLineSinkInfos = handTwoStreamData(beforeCycleDataListMap, beforeCycle.getCycleOrder(),
crossCode, beforeCycle.getBeginDateTime(), beforeCycle.getEndDateTime());
// TODO 只有当前周期有数据sink，才算计算完成，否则继续等待触发
if(travelLineSinkInfos.size()>0){
out.collect(travelLineSinkInfos);
// 当前周期已经处理完成了，kafka中再次来了该周期的数据不需要再次触发计算了
alreadyHandleCycleSignalDataState.put(beforeCycle.getCycleOrder(), beforeCycle);
listToRemove.add(beforeCycle.getCycleOrder());
}
}
}
// 清理待计算的周期数据，计算完成后进行移除
for (Integer i:listToRemove) {
cycleSignalDataState.remove(i);
}
}
// 拿到这个周期内的当前路口的所有的车道的旅行信息数据 并移除之前周期的数据
Map<String, List<TravelInfo>> thisCycleDataListMap = filterThisCycleData(crossCode, beginDateTime, endDateTime);
//如果，有旅行信息数据就计算，否则就不计算
if (thisCycleDataListMap!=null && !thisCycleDataListMap.values().isEmpty()) {
List<TravelLineSinkInfo> travelLineSinkInfos = handTwoStreamData(thisCycleDataListMap, cycleOrder, crossCode, beginDateTime, endDateTime);
// TODO 只有当前周期有数据sink，才算计算完成，否则继续等待触发
if(travelLineSinkInfos.size()>0){
out.collect(travelLineSinkInfos);
// 当前周期已经处理完成了，，kafka中再次来了该周期的数据不需要再次触发计算了
alreadyHandleCycleSignalDataState.put(cycleOrder, value);
cycleSignalDataState.remove(cycleOrder);
}
} else {
cycleSignalDataState.put(cycleOrder, value);
}
// 注册定时器
//获取当前的ProcessingTime
long currentProcessingTime = ctx.timerService().currentProcessingTime();
// 只在某个时间段注册定时器
SimpleDateFormat sdf = new SimpleDateFormat("HH");
if (sdf.format(currentProcessingTime).equals("14")) {
ctx.timerService().registerProcessingTimeTimer(currentProcessingTime + 30 * 60 * 1000);
}
} catch (Exception e) {
e.printStackTrace();
}
}
@Override
public void onTimer(long timestamp, CoProcessFunction<TravelInfo, CycleSignalData, List<TravelLineSinkInfo>>.OnTimerContext ctx, Collector<List<TravelLineSinkInfo>> out) throws Exception {
super.onTimer(timestamp, ctx, out);
dim_cnt_cross_lane_position = getLaneDimData();
}
/**
* 处理2个流的数据，并sink
* @param thisCycleDataListMap
* @param cycleOrder
* @param crossCode
* @throws SQLException
*/
private List<TravelLineSinkInfo> handTwoStreamData(Map<String, List<TravelInfo>> thisCycleDataListMap, Integer cycleOrder,String crossCode
,Long beginDateTime ,Long endDateTime) throws Exception {
Set<String> laneIds = thisCycleDataListMap.keySet();
List<TravelLineSinkInfo> sinkList = new ArrayList<>();
for (String laneId : laneIds) {
// 当前车道的所有的旅行信息 ，一辆车对应1条信息
List<TravelInfo> travelInfos = thisCycleDataListMap.get(laneId);
//需要将这个车道上的所有的车辆，按照进入路口的时间依次排序才能计算出车头时距
List<TravelInfo> orderedTravelInfoList = orderByInCrossTime(travelInfos);
List<TravelCarInfo> list = new ArrayList<>();
//这里的旅行信息的数据，一定是完成了从路口出现到丢失的整个过程的数据，一辆车只有一条数据
//根据周期数据及旅行信息，查询维表，将每一条旅行信息转换为行车情况明细数据(1对1转换)
for (int i = 0; i < orderedTravelInfoList.size(); i++) {
TravelInfo travel = orderedTravelInfoList.get(i);
Long beforeCarInCrossTime;
if (i != 0) {
beforeCarInCrossTime = travelInfos.get(i - 1).getInCrossTime();
} else {
beforeCarInCrossTime = 0L;
}
TravelCarInfo travelCarInfo = trunTravelAndCycleToTravelCarInfo(travel, cycleOrder, beforeCarInCrossTime,beginDateTime,endDateTime);
list.add(travelCarInfo);
}
// list存的是一个周期内的当前车道 旅行信息数据 行车情况的明细数据
// 根据一个车道内一个周期的行车情况的明细数据，计算具体的指标值，这里按照流向进行预聚合统计
//对当前list中的数据按照流向进行拆分
Map<String, List<TravelCarInfo>> map = new HashMap<>();
for (TravelCarInfo t : list) {
List<TravelCarInfo> lineList;
if (!map.containsKey(t.getFlowDirection())) {
lineList = new ArrayList<>();
} else {
lineList = map.get(t.getFlowDirection());
}
lineList.add(t);
map.put(t.getFlowDirection(), lineList);
}
// 按照流向分组完成后，对每个流向的数据进行聚合统计操作，并记录数据，准备写入数据库
Set<String> flowDirections = map.keySet();
for (String flowDirection : flowDirections) {
//当前流向 所有的车辆行车情况信息
List<TravelCarInfo> next = map.get(flowDirection);
//在循环一个车道某个流向过程中需要计算的数据
//1、通过时间之和
//2、每辆车通过平均车速之和
//3、控制延误之和
//4、停车次数之和
//5、停车延误之和
//6、车头时距之和
//7、最后一辆车进入路口的时间（即inCrossTime最大值，记录此值是为了，后续有迟到数据，计算车头时距的时候用的）
Long sumPassTime = 0L;
double sumSpeed = 0.0;
Long sumControlDelay = 0L;
Integer sumStopTimes = 0;
Long sumStopDelay = 0L;
Long sumCarHeadTimeGap = 0L;
Long maxInCrossTime = 0L;
for (TravelCarInfo t : next) {
sumPassTime = sumPassTime + t.getPassTime();
sumSpeed = sumSpeed + t.getAverageSpeed();
sumControlDelay = sumControlDelay + t.getControlDelayTime();
sumStopTimes = sumStopTimes + t.getStopTimes();
sumStopDelay = sumStopDelay + t.getStopDelayTime();
sumCarHeadTimeGap = sumCarHeadTimeGap + t.getCarHeadTimeGap();
if (t.getInCrossTime() > maxInCrossTime) {
maxInCrossTime = t.getInCrossTime();
}
}
int size = next.size();
TravelLineSinkInfo travelLineSinkInfo = new TravelLineSinkInfo();
travelLineSinkInfo.setCross_id(crossCode);
travelLineSinkInfo.setCycle_id(cycleOrder);
travelLineSinkInfo.setDirection(next.get(0).getDirection());
travelLineSinkInfo.setFlow_direction(flowDirection);
travelLineSinkInfo.setLane_id(next.get(0).getInCrossLineId());
travelLineSinkInfo.setPass_numbers(next.size());
travelLineSinkInfo.setAverage_pass_time(sumPassTime/(size*1.000*1000));
travelLineSinkInfo.setAverage_pass_speed(sumSpeed / (size));
travelLineSinkInfo.setAverage_control_delay(-1.0);
travelLineSinkInfo.setAverage_stop_times(sumStopTimes/(size*1.000));
travelLineSinkInfo.setAverage_stop_delay(sumStopDelay/(size*1.000*1000));
travelLineSinkInfo.setAverage_car_head_time_gap(sumCarHeadTimeGap/(size*1.000*1000));
travelLineSinkInfo.setLast_car_inCross_time(maxInCrossTime);
travelLineSinkInfo.setCycle_begin_time(beginDateTime);
travelLineSinkInfo.setCycle_end_time(endDateTime);
sinkList.add(travelLineSinkInfo);
}
}
return sinkList ;
// sinkTravelLineSinkInfoToMysql(sinkList);
}
/**
* 过滤出旅行信息状态中，属于这个周期内的计算数据，返回，对于已经过期的数据进行清理
* @param beginDateTime
* @param endDateTime
* @return
*/
private Map<String,List<TravelInfo>> filterThisCycleData(String crossCode,Long beginDateTime ,Long endDateTime) throws Exception {
Map<String, List<TravelInfo>> stringListMap = travelInfoState.get(crossCode);
Map<String, List<TravelInfo>> thisCycleData = new HashMap<>();
Map<String, List<TravelInfo>> leftCycleData = new HashMap<>();
if (stringListMap != null) {
//所有的车道数据
Set<String> laneIds = stringListMap.keySet();
for (String laneId : laneIds) {
List<TravelInfo> allTravelInfos = stringListMap.get(laneId);
List<TravelInfo> leftData = new ArrayList<>(); //剩下的数据，不在，当前周期的，属于下个周期的
List<TravelInfo> thisData = new ArrayList<>(); //属于当前周期的数据
for (TravelInfo t : allTravelInfos) {
//进入路口的时间属于这个周期
if (t.getInCrossTime() >= beginDateTime && t.getInCrossTime() <= endDateTime && t.getLocations()!=null) {
//当前周期用到的数据
thisData.add(t);
//如果，进入路口时距小于周期开始时间，且已经超过30min，则清理这个数据
}
else if(t.getInCrossTime()<beginDateTime && beginDateTime - t.getInCrossTime() > 1000*60*60*3 ) {
logger.info("移除当前旅行信息-超过30分钟都没有触发计算"+"车辆id"+t.getCarId() + "进入路口时间："+sdf.format(t.getInCrossTime()));
}
else {
//当前周期没有用的的数据
leftData.add(t);
}
}
if(thisData.size()!=0){
thisCycleData.put(laneId, thisData);
}
leftCycleData.put(laneId,leftData);
}
//更新旅行信息状态中的数据，只保留没有使用的旅行信息数据
travelInfoState.put(crossCode, leftCycleData);
return thisCycleData;
}
return null;
}
private List<TravelInfo> orderByInCrossTime(List<TravelInfo> travelInfos) {
List<TravelInfo> travelInfoList = new ArrayList<>();
Set<TravelInfo> set = new TreeSet<>();
for (TravelInfo t:travelInfos) {
set.add(t);
}
travelInfoList.addAll(set);
return travelInfoList ;
}
/**
* 将旅行信息转换为行车情况明细信息数据
*
* @param travel
* @return
*/
private TravelCarInfo trunTravelAndCycleToTravelCarInfo(TravelInfo travel, Integer cycleOrder, Long beforeCarInCrossTime,Long beginDateTime ,Long endDateTime) throws Exception {
//1、计算进入时间
long passTime = travel.getInCrossTime() - travel.getArrivedTime();
//2、获取方向
String direction = dim_cnt_cross_lane_position.get(travel.getCrossId()).get(travel.getInCrossLineId()+"direction");
//2.1 获取流向
String flow_direction = getFlowDirection(travel) ;
//3、获取平均速度 只看在旅行开始时间和旅行结束时间之间的数据
Double averageSpeed = getAverageSpeed(travel.getLocations(),travel.getTravelBeginTime(),travel.getTravelEndTime());
//5、计算停车次数(这里使用停车次数，非实际停车次数)
Integer stopTimes = getStopTimes(travel.getLocations());
//6、计算停车延误时间
Long stopDelayTime = getStopDelayTime(travel.getLocations());
//4、控制延迟时间 暂时不算,使用停车延误时间
Long controlDelayTime = -stopDelayTime;
//7、计算车头时距 要分车道
Long carHeadTimeGap;
if (beforeCarInCrossTime == 0L) {
carHeadTimeGap = 0L;
} else {
// 取绝对值，避免出现插队导致车头时距变成负数
carHeadTimeGap = Math.abs(travel.getInCrossTime() - beforeCarInCrossTime);
}
TravelCarInfo travelCarInfo = new TravelCarInfo(travel.getCrossId(), cycleOrder,
travel.getCarId()
, passTime,travel.getInCrossTime(), travel.getInCrossLineId(), direction, flow_direction,averageSpeed,
controlDelayTime, stopTimes, stopDelayTime, carHeadTimeGap);
return travelCarInfo;
}
/**
* 查询车道维表数据-数据存在map中，key是路口编号，
*
* @return
* @throws SQLException
*/
private Map<String, Map<String, String>> getLaneDimData() throws Exception {
connection=DorisUtils.getConnection();
String sql = "select cross_id, lane_id,position,turn_direction as flow_direction from dim_cnt_cross_lane_position ";
PreparedStatement ps = connection.prepareStatement(sql);
ResultSet resultSet = ps.executeQuery();
Map<String, Map<String, String>> dimData = new HashMap<>();
while (resultSet.next()) {
//当前这个路口的车道维度数据不存在
if (dimData.get(resultSet.getString("cross_id")) == null) {
Map<String, String> map = new HashMap<>();
map.put(resultSet.getString("lane_id")+"direction", resultSet.getString("position"));
map.put(resultSet.getString("lane_id")+"flow_direction", resultSet.getString("flow_direction"));
dimData.put(resultSet.getString("cross_id"), map);
} else {
Map<String, String> map = dimData.get(resultSet.getString("cross_id"));
map.put(resultSet.getString("lane_id")+"direction", resultSet.getString("position"));
map.put(resultSet.getString("lane_id")+"flow_direction", resultSet.getString("flow_direction"));
dimData.put(resultSet.getString("cross_id"), map);
}
}
if(connection!=null){
connection.close();
}
return dimData;
}
/**
* 计算平均速度，传入一个list集合，计算每个元素中速度的平均值
* 只看旅行期间的平均速度，（照片时间在旅行开始和结束之间）
* @param locations
* @return
*/
private Double getAverageSpeed(List<Location> locations, Long travelBeginTime, Long travelEndTime) {
Double sumSpeed = 0.0;
if(travelBeginTime!=null && travelEndTime!=null) {
int i =0 ;
for (Location location : locations) {
if(location.getDtTranjectory()>=travelBeginTime && location.getDtTranjectory()<=travelEndTime) {
sumSpeed = location.getSpeed() + sumSpeed;
i++;
}
}
return sumSpeed / i;
//如果，没有旅行开始和结束时间，就算全部的
} else {
for (Location location : locations) {
sumSpeed = location.getSpeed() + sumSpeed;
}
return sumSpeed / locations.size();
}
}
/**
* 计算停车次数 根据传入的locations集合，划分到不同的周期中去，在每个周期统计实际停车次数
* 根据捕获时间排序，从小到大
* 这里求的是实际停车次数
* @param locations
* @return
*/
private Integer getStopTimes(List<Location> locations) throws Exception {
//周期的开始时间
Iterator<Long> iterator1 = cycleSectionState.get().iterator();
Set<Long> set0 = new TreeSet<>();
List<Long> beginTimes ; // 最近3个周期开始时间
while(iterator1.hasNext()){
set0.add(iterator1.next());
}
List<Long> list = new ArrayList<>();
for (Long l:set0) {
list.add(l);
}
// 这个会对周期切片的数据进行更新操作，最多保留最近3个周期的时间片
if(list.size() > 3) {
beginTimes = list.subList(list.size() - 3, list.size());
cycleSectionState.update(beginTimes);
} else {
beginTimes = list;
}
// 根据最近的几个周期的开始时间对locations数据进行划分，划分到不同的周期
Map<Long, List<Location>> longListMap = divideLocations(beginTimes, locations);
Set<Long> longs = longListMap.keySet();
int stopTimes =0 ;
for (Long l:longs) {
List<Location> locations1 = longListMap.get(l);
Integer realStopTimes = getRealStopTimes(locations1);
// 一个周期内发生的实际停车次数大于0,停车次数加1
if(realStopTimes > 0){
stopTimes++;
}
}
return stopTimes;
}
/**
* 根据一系列的location信息，判断发生的实际停车次数
* @param locations
* @return
*/
private Integer getRealStopTimes(List<Location> locations){
// 按照捕获时间进行排序
Set<Location> set = new TreeSet<>();
for (Location location : locations) {
set.add(location);
}
Iterator<Location> iterator = set.iterator();
List<Location> speedList = new ArrayList<>();
while (iterator.hasNext()) {
speedList.add(iterator.next());
}
int realStopTimes = 0;
for (int i = 0; i < speedList.size(); i++) {
//第一帧数据小于3视为停车
if (i == 0 && speedList.get(i).getSpeed() < 3.0) {
realStopTimes++;
//从第二帧开始，只有前一帧速度大于10，且这一帧数据小于3的才算一次停车，连续小于3的视为一次停车
} else {
if (speedList.get(i).getSpeed() < 3.0 && speedList.get(i - 1).getSpeed() >= 10.0) {
realStopTimes++;
}
}
}
return realStopTimes;
}
/**
* 根据最近的几个周期的开始时间，把对应的locations划分到不同的周期中去，以便计算停车次数
* @param beginTimes
* @param locations
* @return 返回值是map结构 key->list<Location> 周期开始时间 -> 该周期对应的位置信息
*/
private Map<Long,List<Location>> divideLocations(List<Long> beginTimes,List<Location> locations){
Map<Long,List<Location>> map = new HashMap<>();
if(beginTimes.size()==1){
List<Location> list = new ArrayList<>();
List<Location> list2 = new ArrayList<>();
for (Location location:locations) {
if (location.getDtTranjectory()<beginTimes.get(0)){
list.add(location);
} else {
list2.add(location);
}
}
map.put(beginTimes.get(0),list);
map.put(999999999L,list2);
}
for (int i=0;i<beginTimes.size();i++) {
List<Location> list = new ArrayList<>();
if(i==0){
for (Location location:locations) {
if (location.getDtTranjectory()<beginTimes.get(i)){
list.add(location);
}
}
map.put(beginTimes.get(i),list);
// 最后一个周期的数据
} else if(i==(beginTimes.size()-1)){
for (Location location:locations) {
if (location.getDtTranjectory()>beginTimes.get(i)){
list.add(location);
}
}
map.put(beginTimes.get(i),list);
} else {
for (Location location:locations) {
if (location.getDtTranjectory()<beginTimes.get(i)&&location.getDtTranjectory()>beginTimes.get(i-1)){
list.add(location);
}
}
map.put(beginTimes.get(i),list);
}
}
return map ;
}
/**
* 计算停车延误时间 根据传入的locations集合，通过按序排列每帧信息，计算正常行驶开始时间置上一个停止时间的时间差，进行累计即为累计的停车延误时间
* 根据捕获时间排序，从小到大
*
* @param locations
* @return
*/
private Long getStopDelayTime(List<Location> locations) {
Set<Location> set = new TreeSet<>();
for (Location location : locations) {
set.add(location);
}
// 按照捕获时间进行排序
Iterator<Location> iterator = set.iterator();
List<Location> speedList = new ArrayList<>();
while (iterator.hasNext()) {
Location next = iterator.next();
speedList.add(next);
}
Long stopDelayTime = 0L;
Long stopStart = 0L;
for (int i = 0; i < speedList.size(); i++) {
//第一帧数据小于3视为停车
if (i == 0 && speedList.get(i).getSpeed() < 3.0) {
stopStart = speedList.get(i).getDtTranjectory();
//从第二帧开始，只有前一帧速度大于10，且这一帧数据小于10的才算一次停车，连续小于3的视为一次停车
} else {
//发生停车事件 车速小于3.0视为停车
// if (speedList.get(i).getSpeed() < 3.0 && speedList.get(i - 1).getSpeed() >= 10.0) {
if (speedList.get(i).getSpeed() < 3.0 && speedList.get(i - 1).getSpeed() >= 3.0) {
stopStart = speedList.get(i).getDtTranjectory();
}
//开始正常行驶
else {
if (stopStart != 0L && speedList.get(i).getSpeed() >= 3.0) {
stopDelayTime = (speedList.get(i).getDtTranjectory() - stopStart) + stopDelayTime;
stopStart = 0L;
}
// 如果，是最后一帧数据并且是停止状态，且前面也是停止状态，则进行计算延迟时间
if (i == speedList.size() - 1 && speedList.get(i).getSpeed() < 3.0 && stopStart != 0L) {
stopDelayTime = (speedList.get(i).getDtTranjectory() - stopStart) + stopDelayTime;
}
}
}
}
return stopDelayTime;
}
/**
* 根据进入路口车道，和驶出路口车道结合起来，判断具体的流向
* 根据进入路口的方法向和驶出路口的方向，判断流向
* @return
*/
private String getFlowDirection(TravelInfo travel){
//根据车道id，查询方向
String inLineId = travel.getInCrossLineId()!=null?travel.getInCrossLineId():travel.getArrivedLineId();
String outLineId = travel.getOutCrossLineId()!=null?travel.getOutCrossLineId():travel.getAwayLineId();
String inDirection = dim_cnt_cross_lane_position.get(travel.getCrossId()).get(inLineId+"direction");
String outDirection = dim_cnt_cross_lane_position.get(travel.getCrossId()).get(outLineId+"direction");
if(inDirection==null || outDirection==null) {
String flowDirection = travel.getFlowDirection();
if(flowDirection==null){
return "未知";
} else {
String[] split = flowDirection.split("-");
inDirection=split[0];
outDirection=split[1];
}
}
if(inDirection!=null&&inDirection.equals(outDirection)){
return inDirection+"掉头";
} else {
switch (inDirection){
case "东":
if(outDirection.equals("南")){return "东左转";}
if(outDirection.equals("西")){return "东直行";}
if(outDirection.equals("北")){return "东右转";}
case "南":
if(outDirection.equals("东")){return "南右转";}
if(outDirection.equals("西")){return "南左转";}
if(outDirection.equals("北")){return "南直行";}
case "西":
if(outDirection.equals("东")){return "西直行";}
if(outDirection.equals("南")){return "西右转";}
if(outDirection.equals("北")){return "西左转";}
case "北":
if(outDirection.equals("东")){return "北左转";}
if(outDirection.equals("南")){return "北直行";}
if(outDirection.equals("西")){return "北右转";}
}
return "未知";
}
}
}
realtime/hologram-streaming/src/main/java/com/zhht/irn/functions/TravelEventAndCycleCoProcessFunction.java
...@@ -38,9 +38,10 @@ public class TravelEventAndCycleCoProcessFunction extends CoProcessFunction<Trav ...@@ -38,9 +38,10 @@ public class TravelEventAndCycleCoProcessFunction extends CoProcessFunction<Trav
private MapState<String, Map<String, Map<String,List<TravelEvent>>>> leftDataEventState; //缓存当前周期没有进入路口的车辆的旅行事件 private MapState<String, Map<String, Map<String,List<TravelEvent>>>> leftDataEventState; //缓存当前周期没有进入路口的车辆的旅行事件
private MapState<String, TravelEvent> outCrossEventState; //缓存驶出路口的旅行事件 private MapState<String, TravelEvent> outCrossEventState; //缓存驶出路口的旅行事件
private MapState<Integer, CycleSignalData> cycleSignalDataState; //缓存周期数据（避免周期数据先到，而旅行数数据后到的情况） private MapState<Integer, CycleSignalData> cycleSignalDataState; //缓存周期数据（避免周期数据先到，而旅行数数据后到的情况）
private MapState<Integer, CycleSignalData> alreadyHandleCycleSignalDataState; //已经处理过的周期数据，避免重复触发
private MapState<String, Long> carRecordState; //记录车辆的 车辆id --> 进入时间 （后面，根据这个时间，和车辆id，移除过期车辆的数据 private MapState<String, Long> carRecordState; //记录车辆的 车辆id --> 进入时间 （后面，根据这个时间，和车辆id，移除过期车辆的数据
private DruidDataSource dataSource; // private DruidDataSource dataSource;
private transient DruidPooledConnection connection; private transient DruidPooledConnection connection;
private static Map<String, Map<String, String>> dim_cnt_cross_lane_position; private static Map<String, Map<String, String>> dim_cnt_cross_lane_position;
...@@ -64,6 +65,10 @@ public class TravelEventAndCycleCoProcessFunction extends CoProcessFunction<Trav ...@@ -64,6 +65,10 @@ public class TravelEventAndCycleCoProcessFunction extends CoProcessFunction<Trav
new MapStateDescriptor("cycleSignalDataState", new MapStateDescriptor("cycleSignalDataState",
TypeInformation.of(Integer.class), TypeInformation.of(CycleSignalData.class)); TypeInformation.of(Integer.class), TypeInformation.of(CycleSignalData.class));
MapStateDescriptor<Integer, CycleSignalData> alreadyHandleCycleSignalDataStateDescriptor =
new MapStateDescriptor("alreadyHandleCycleSignalDataState",
TypeInformation.of(Integer.class), TypeInformation.of(CycleSignalData.class));
MapStateDescriptor<String, Long> carRecordStateDescriptor = MapStateDescriptor<String, Long> carRecordStateDescriptor =
new MapStateDescriptor("carRecordState", new MapStateDescriptor("carRecordState",
TypeInformation.of(String.class), TypeInformation.of(Long.class)); TypeInformation.of(String.class), TypeInformation.of(Long.class));
...@@ -78,36 +83,58 @@ public class TravelEventAndCycleCoProcessFunction extends CoProcessFunction<Trav ...@@ -78,36 +83,58 @@ public class TravelEventAndCycleCoProcessFunction extends CoProcessFunction<Trav
// 过期对象的清理策略 全量清理 // 过期对象的清理策略 全量清理
.cleanupFullSnapshot() .cleanupFullSnapshot()
.build(); .build();
carRecordStateDescriptor.enableTimeToLive(stateTtlConfig);
carRecordStateDescriptor.enableTimeToLive(stateTtlConfig); //设置车辆过期时间
StateTtlConfig stateTtlConfig2 = StateTtlConfig
// 状态有效时间 300min
.newBuilder(Time.minutes(300))
// 设置状态的更新类型
.setUpdateType(StateTtlConfig.UpdateType.OnCreateAndWrite)
// 已过期还未被清理掉的状态数据不返回给用户
.setStateVisibility(StateTtlConfig.StateVisibility.NeverReturnExpired)
// 过期对象的清理策略 全量清理
.cleanupFullSnapshot()
.build();
alreadyHandleCycleSignalDataStateDescriptor.enableTimeToLive(stateTtlConfig2); //设置已处理周期过期时间
travelEventState = getRuntimeContext().getMapState(travelEventStateDescriptor); travelEventState = getRuntimeContext().getMapState(travelEventStateDescriptor);
leftDataEventState = getRuntimeContext().getMapState(leftDataEventStateDescriptor); leftDataEventState = getRuntimeContext().getMapState(leftDataEventStateDescriptor);
outCrossEventState = getRuntimeContext().getMapState(outCrossEventStateDescriptor); outCrossEventState = getRuntimeContext().getMapState(outCrossEventStateDescriptor);
cycleSignalDataState = getRuntimeContext().getMapState(cycleSignalDataStateDescriptor); cycleSignalDataState = getRuntimeContext().getMapState(cycleSignalDataStateDescriptor);
alreadyHandleCycleSignalDataState = getRuntimeContext().getMapState(alreadyHandleCycleSignalDataStateDescriptor);
carRecordState = getRuntimeContext().getMapState(carRecordStateDescriptor); carRecordState = getRuntimeContext().getMapState(carRecordStateDescriptor);
/* dataSource = DruidConnectPoolUtils.getDataSource
connection = DruidConnectPoolUtils.getConnection(); ("jdbc:mysql://10.243.0.22:9030/qxlk?characterEncoding=utf8",
// ("jdbc:mysql://localhost:9030/demo?characterEncoding=utf8", // ("jdbc:mysql://localhost:9030/qxlk?characterEncoding=utf8",
// ("jdbc:mysql://localhost:3307/test?characterEncoding=utf8", // ("jdbc:mysql://localhost:3307/test?characterEncoding=utf8",
// ("jdbc:mysql://10.243.0.26:3306/test?characterEncoding=utf8", // ("jdbc:mysql://10.243.0.26:3306/test?characterEncoding=utf8",
/* ("jdbc:mysql://10.243.0.22:9030/demo?characterEncoding=utf8", // ("jdbc:mysql://10.243.0.22:9030/demo?characterEncoding=utf8",
"root", "root",
// "root123"); // "root123");
"mima"); "mima");
*/
// dataSource.getConnection(); connection = dataSource.getConnection();*/
connection=DruidConnectPoolUtils.getConnection();
dim_cnt_cross_lane_position = getLaneDimData(connection); dim_cnt_cross_lane_position = getLaneDimData(connection);
} }
@Override
public void close() throws Exception {
super.close();
if(connection!=null){
connection.close();
}
}
//把每个路口的数据 车辆旅行信息数据，存储在状态中，当周期数据进来的时候，再进行触发计算 //把每个路口的数据 车辆旅行信息数据，存储在状态中，当周期数据进来的时候，再进行触发计算
@Override @Override
public void processElement1(TravelEvent value, CoProcessFunction<TravelEvent, CycleSignalData, Object>.Context ctx, public void processElement1(TravelEvent value, CoProcessFunction<TravelEvent, CycleSignalData, Object>.Context ctx,
Collector<Object> out) throws Exception { Collector<Object> out) throws Exception {
System.out.println("处理的数据有"+value.getEventType()); // System.out.println("处理的数据有"+value.getEventType());
//记录当前车辆的进入路口的时间，车辆id --> 进入时间 （后面，根据这个时间，和车辆id，移除过期车辆的数据） //记录当前车辆的进入路口的时间，车辆id --> 进入时间 （后面，根据这个时间，和车辆id，移除过期车辆的数据）
if("INCROSS".equals(value.getEventType())){ if("INCROSS".equals(value.getEventType())){
...@@ -192,6 +219,12 @@ public class TravelEventAndCycleCoProcessFunction extends CoProcessFunction<Trav ...@@ -192,6 +219,12 @@ public class TravelEventAndCycleCoProcessFunction extends CoProcessFunction<Trav
public void processElement2(CycleSignalData value, CoProcessFunction<TravelEvent, CycleSignalData, Object>.Context ctx, public void processElement2(CycleSignalData value, CoProcessFunction<TravelEvent, CycleSignalData, Object>.Context ctx,
Collector<Object> out) throws Exception { Collector<Object> out) throws Exception {
//判断当前周期是否执行过了,存在周期数据重复下发的情况，只触发一次计算
if(alreadyHandleCycleSignalDataState.contains(value.getCycleOrder())){
System.out.println("周期数据重复----当前路口id是"+value.getCrossCode()+"当前周期已经计算完成了"+value.getCycleOrder());
return;
}
// 注册定时器 // 注册定时器
//获取当前的ProcessingTime //获取当前的ProcessingTime
long currentProcessingTime = ctx.timerService().currentProcessingTime(); long currentProcessingTime = ctx.timerService().currentProcessingTime();
...@@ -210,7 +243,7 @@ public class TravelEventAndCycleCoProcessFunction extends CoProcessFunction<Trav ...@@ -210,7 +243,7 @@ public class TravelEventAndCycleCoProcessFunction extends CoProcessFunction<Trav
List<Integer> cycles = new ArrayList<>(); List<Integer> cycles = new ArrayList<>();
while(iterator.hasNext()){ while(iterator.hasNext()){
CycleSignalData next = iterator.next(); CycleSignalData next = iterator.next();
System.out.println("当前处理周期"+next.getCycleOrder()+"<---->路口"+next.getCrossCode()); // System.out.println("当前处理周期"+next.getCycleOrder()+"<---->路口"+next.getCrossCode());
handleTwoStream(next); handleTwoStream(next);
cycles.add(next.getCycleOrder()); cycles.add(next.getCycleOrder());
} }
...@@ -222,14 +255,13 @@ public class TravelEventAndCycleCoProcessFunction extends CoProcessFunction<Trav ...@@ -222,14 +255,13 @@ public class TravelEventAndCycleCoProcessFunction extends CoProcessFunction<Trav
// 如果，周期数据先到，旅行事件数据后到的话，把周期数据暂存到状态中去， // 如果，周期数据先到，旅行事件数据后到的话，把周期数据暂存到状态中去，
} else { } else {
System.out.println("路口暂无旅行事件信息-等待下次触发"); // System.out.println(value.getCrossCode()+"路口暂无旅行事件信息-等待下次触发");
cycleSignalDataState.put(cycleOrder,value); cycleSignalDataState.put(cycleOrder,value);
} }
} }
private void handleTwoStream(CycleSignalData value) throws Exception { private void handleTwoStream(CycleSignalData value) throws Exception {
System.out.println("处理的数据有" + value.getCrossCode()+"--->"+value.getCycleOrder());
String crossCode = value.getCrossCode(); String crossCode = value.getCrossCode();
Integer cycleOrder = value.getCycleOrder(); Integer cycleOrder = value.getCycleOrder();
...@@ -240,8 +272,6 @@ public class TravelEventAndCycleCoProcessFunction extends CoProcessFunction<Trav ...@@ -240,8 +272,6 @@ public class TravelEventAndCycleCoProcessFunction extends CoProcessFunction<Trav
//travelEventState 是一个map结构，key是每一个路口id，根据路口id，获取到当前路口的所有的数据 //travelEventState 是一个map结构，key是每一个路口id，根据路口id，获取到当前路口的所有的数据
Map<String, Map<String, List<TravelEvent>>> crossMap = travelEventState.get(crossCode); Map<String, Map<String, List<TravelEvent>>> crossMap = travelEventState.get(crossCode);
// System.out.println("旅行事件数据有：" + crossMap);
// //
Map<String, Map<String, List<TravelEvent>>> leftDataMap = leftDataEventState.get(crossCode); Map<String, Map<String, List<TravelEvent>>> leftDataMap = leftDataEventState.get(crossCode);
// System.out.println("上个周期剩余未计算的数据有：" + leftDataMap); // System.out.println("上个周期剩余未计算的数据有：" + leftDataMap);
...@@ -269,10 +299,13 @@ public class TravelEventAndCycleCoProcessFunction extends CoProcessFunction<Trav ...@@ -269,10 +299,13 @@ public class TravelEventAndCycleCoProcessFunction extends CoProcessFunction<Trav
} }
//如果，一条数据都没sink，说明周期数据先来的，事件数据迟到了 //如果，一条数据都没sink，说明周期数据先来的，事件数据迟到了
if(sinkCounts==0){ if(sinkCounts==0){
System.out.println("当前路口"+crossCode+"当前周期，没有sink的数据"+cycleOrder); // System.out.println("当前路口"+crossCode+"当前周期，没有sink的数据"+cycleOrder);
// cycleSignalDataState.put(cycleOrder,value); cycleSignalDataState.put(cycleOrder,value);
} }
// 当前周期已经处理完成了，，kafka中再次来了该周期的数据不需要再次触发计算了
alreadyHandleCycleSignalDataState.put(cycleOrder,value);
} }
} }
...@@ -289,7 +322,7 @@ public class TravelEventAndCycleCoProcessFunction extends CoProcessFunction<Trav ...@@ -289,7 +322,7 @@ public class TravelEventAndCycleCoProcessFunction extends CoProcessFunction<Trav
super.onTimer(timestamp, ctx, out); super.onTimer(timestamp, ctx, out);
System.out.println("触发定时器了。。。。"); // System.out.println("触发定时器了。。。。");
Iterator<String> carIds = carRecordState.keys().iterator(); Iterator<String> carIds = carRecordState.keys().iterator();
Iterator<Long> eventTimes = carRecordState.values().iterator(); Iterator<Long> eventTimes = carRecordState.values().iterator();
// 获取到最新的的到时间，以便作为参考依据移除过期数据，这里不使用系统的或者服务器的时间的原因，是避免服务器时间与外界数据时间不一致的情况 // 获取到最新的的到时间，以便作为参考依据移除过期数据，这里不使用系统的或者服务器的时间的原因，是避免服务器时间与外界数据时间不一致的情况
...@@ -322,11 +355,18 @@ public class TravelEventAndCycleCoProcessFunction extends CoProcessFunction<Trav ...@@ -322,11 +355,18 @@ public class TravelEventAndCycleCoProcessFunction extends CoProcessFunction<Trav
//清空当前车辆的数据 //清空当前车辆的数据
for (String carId:needRemoveCars) { for (String carId:needRemoveCars) {
next1.remove(carId); next1.remove(carId);
System.out.println("移除当前车辆数据"+carId);
} }
} }
} }
//更新维表数据 只在每天12点的时候更新
long l = System.currentTimeMillis();
SimpleDateFormat sdf = new SimpleDateFormat("HH");
if(sdf.format(l).equals("12")){
dim_cnt_cross_lane_position = getLaneDimData(connection);
}
} }
/** /**
...@@ -353,8 +393,8 @@ public class TravelEventAndCycleCoProcessFunction extends CoProcessFunction<Trav ...@@ -353,8 +393,8 @@ public class TravelEventAndCycleCoProcessFunction extends CoProcessFunction<Trav
Map<String, List<TravelEvent>> leftLineData; Map<String, List<TravelEvent>> leftLineData;
if(leftDataCross!=null){ if(leftDataCross!=null){
leftDataCross = leftDataEventState.get(crossCode); leftDataCross = leftDataEventState.get(crossCode);
System.out.println("当前路口是"+crossCode+"当前车道id是"+lineId); /* System.out.println("当前路口是"+crossCode+"当前车道id是"+lineId);
System.out.println("当前路口,当前车道存在未计算的数据有"+leftDataCross.get(lineId)); System.out.println("当前路口,当前车道存在未计算的数据有"+leftDataCross.get(lineId));*/
leftLineData = leftDataCross.get(lineId); leftLineData = leftDataCross.get(lineId);
} else { } else {
leftLineData = new HashMap<>(); leftLineData = new HashMap<>();
...@@ -407,7 +447,7 @@ public class TravelEventAndCycleCoProcessFunction extends CoProcessFunction<Trav ...@@ -407,7 +447,7 @@ public class TravelEventAndCycleCoProcessFunction extends CoProcessFunction<Trav
// ，后面触发计算时，进行合并到新来的数据暂存状态中去） // ，后面触发计算时，进行合并到新来的数据暂存状态中去）
//TODO 处理无法完成计算的数据 事件没有到齐的情况 //TODO 处理无法完成计算的数据 事件没有到齐的情况
if(!allEventOfCar.containsKey("INCROSS")||!allEventOfCar.containsKey("ARRIVED")){ if(!allEventOfCar.containsKey("INCROSS")||!allEventOfCar.containsKey("ARRIVED")){
System.out.println("当前数据不完整，无法计算。。。。。"+carId); // System.out.println("当前数据不完整，无法计算。。。。。"+carId);
//如果，当前车道 存在，不能完成计算的数据 （数据不全，没有进入路口的数据，则等待下次触发计算） //如果，当前车道 存在，不能完成计算的数据 （数据不全，没有进入路口的数据，则等待下次触发计算）
//获取当前车道 //获取当前车道
String eventLineId = allEventOfCar.values().iterator().next().getEventLineId(); String eventLineId = allEventOfCar.values().iterator().next().getEventLineId();
...@@ -433,8 +473,6 @@ public class TravelEventAndCycleCoProcessFunction extends CoProcessFunction<Trav ...@@ -433,8 +473,6 @@ public class TravelEventAndCycleCoProcessFunction extends CoProcessFunction<Trav
continue; continue;
} }
System.out.println("当前车辆所有事件数据有：==========》\r\n"+allEventOfCar);
//1、通过时间 //1、通过时间
Long passTime = allEventOfCar.get("INCROSS").getEventTime()-allEventOfCar.get("ARRIVED").getEventTime(); Long passTime = allEventOfCar.get("INCROSS").getEventTime()-allEventOfCar.get("ARRIVED").getEventTime();
...@@ -506,12 +544,8 @@ public class TravelEventAndCycleCoProcessFunction extends CoProcessFunction<Trav ...@@ -506,12 +544,8 @@ public class TravelEventAndCycleCoProcessFunction extends CoProcessFunction<Trav
List<TravelLineSinkInfo> sinkList = new ArrayList<>(); List<TravelLineSinkInfo> sinkList = new ArrayList<>();
for (String flowDirection:flowDirectionSet) { for (String flowDirection:flowDirectionSet) {
System.out.println("当前数据流向是------》"+flowDirection);
//一个流向上每个车辆整合后的数据 //一个流向上每个车辆整合后的数据
Set<TravelCarInfo> travelCarInfosOfOneFlow = flowMap.get(flowDirection); Set<TravelCarInfo> travelCarInfosOfOneFlow = flowMap.get(flowDirection);
// sum // sum
//在循环一个车道某个流向过程中需要计算的数据 //在循环一个车道某个流向过程中需要计算的数据
//1、通过时间之和 //1、通过时间之和
...@@ -580,9 +614,8 @@ public class TravelEventAndCycleCoProcessFunction extends CoProcessFunction<Trav ...@@ -580,9 +614,8 @@ public class TravelEventAndCycleCoProcessFunction extends CoProcessFunction<Trav
} }
Map<String, Map<String, List<TravelEvent>>> leftDataMap = leftDataEventState.get(crossCode); // Map<String, Map<String, List<TravelEvent>>> leftDataMap = leftDataEventState.get(crossCode);
System.out.println("这个周期剩余未计算的数据有："+leftDataMap); // System.out.println("这个周期剩余未计算的数据有："+leftDataMap);
return sinkList ; return sinkList ;
} }
...@@ -601,8 +634,6 @@ public class TravelEventAndCycleCoProcessFunction extends CoProcessFunction<Trav ...@@ -601,8 +634,6 @@ public class TravelEventAndCycleCoProcessFunction extends CoProcessFunction<Trav
return map; return map;
} }
//SELECT id, id_old, lane_id, cross_id, name, out_in_type, `position`, out_in_name, lane_length //SELECT id, id_old, lane_id, cross_id, name, out_in_type, `position`, out_in_name, lane_length
//FROM test.dim_cnt_cross_lane_position; //FROM test.dim_cnt_cross_lane_position;
...@@ -634,7 +665,7 @@ public class TravelEventAndCycleCoProcessFunction extends CoProcessFunction<Trav ...@@ -634,7 +665,7 @@ public class TravelEventAndCycleCoProcessFunction extends CoProcessFunction<Trav
dimData.put(resultSet.getString("cross_id"), map); dimData.put(resultSet.getString("cross_id"), map);
} }
} }
connection.close(); // connection.close();
return dimData; return dimData;
} }
...@@ -688,8 +719,6 @@ public class TravelEventAndCycleCoProcessFunction extends CoProcessFunction<Trav ...@@ -688,8 +719,6 @@ public class TravelEventAndCycleCoProcessFunction extends CoProcessFunction<Trav
* @return * @return
*/ */
private String getFlowDirection(Map<String, TravelEvent> allEventOfCar) throws Exception { private String getFlowDirection(Map<String, TravelEvent> allEventOfCar) throws Exception {
// 进来的数据都有INCROSS // 进来的数据都有INCROSS
TravelEvent inCross = allEventOfCar.get("INCROSS"); TravelEvent inCross = allEventOfCar.get("INCROSS");
// 驶出路口的数据，单独存放在一个状态存储中 // 驶出路口的数据，单独存放在一个状态存储中
...@@ -697,7 +726,6 @@ public class TravelEventAndCycleCoProcessFunction extends CoProcessFunction<Trav ...@@ -697,7 +726,6 @@ public class TravelEventAndCycleCoProcessFunction extends CoProcessFunction<Trav
// 当前车辆存在驶出路口的事件 // 当前车辆存在驶出路口的事件
if(outCross!=null){ if(outCross!=null){
//根据车道id，查询方向 //根据车道id，查询方向
String inLineId = inCross.getEventLineId()!=null?inCross.getEventLineId():inCross.getEventLineId(); String inLineId = inCross.getEventLineId()!=null?inCross.getEventLineId():inCross.getEventLineId();
String outLineId = outCross.getEventLineId()!=null?outCross.getEventLineId():outCross.getEventLineId(); String outLineId = outCross.getEventLineId()!=null?outCross.getEventLineId():outCross.getEventLineId();
String inDirection = dim_cnt_cross_lane_position.get(inCross.getCrossId()).get(inLineId+"direction"); String inDirection = dim_cnt_cross_lane_position.get(inCross.getCrossId()).get(inLineId+"direction");
...@@ -750,9 +778,8 @@ public class TravelEventAndCycleCoProcessFunction extends CoProcessFunction<Trav ...@@ -750,9 +778,8 @@ public class TravelEventAndCycleCoProcessFunction extends CoProcessFunction<Trav
*/ */
private void sinkTravelLineSinkInfoToMysql(List<TravelLineSinkInfo> list) throws SQLException { private void sinkTravelLineSinkInfoToMysql(List<TravelLineSinkInfo> list) throws SQLException {
DruidPooledConnection connection = DruidConnectPoolUtils.getConnection();
String sql = "INSERT INTO demo.app_cross_line_travel_car_info" + String sql = "INSERT INTO qxlk.app_cross_line_travel_car_info" +
"(record_date,cross_id,statistic_time,cycle_begin_time,cycle_end_time, lane_id, cycle_order, direction," + "(record_date,cross_id,statistic_time,cycle_begin_time,cycle_end_time, lane_id, cycle_order, direction," +
" flow_direction, pass_numbers," + " flow_direction, pass_numbers," +
" last_car_inCross_time, average_pass_time, average_pass_speed, average_control_delay, " + " last_car_inCross_time, average_pass_time, average_pass_speed, average_control_delay, " +
...@@ -766,10 +793,11 @@ public class TravelEventAndCycleCoProcessFunction extends CoProcessFunction<Trav ...@@ -766,10 +793,11 @@ public class TravelEventAndCycleCoProcessFunction extends CoProcessFunction<Trav
SimpleDateFormat sdf = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss"); SimpleDateFormat sdf = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss");
SimpleDateFormat sdf2 = new SimpleDateFormat("yyyy-MM-dd"); SimpleDateFormat sdf2 = new SimpleDateFormat("yyyy-MM-dd");
String statistic_time = sdf.format(date); String statistic_time = sdf.format(date);
String record_date = sdf2.format(date);
for (TravelLineSinkInfo t :list) { for (TravelLineSinkInfo t :list) {
// 使用事件数据的时间划分数据分区，这样可以进行补数操作
String record_date = sdf2.format(t.getLast_car_inCross_time());
preparedStatement.setString(1, record_date); preparedStatement.setString(1, record_date);
preparedStatement.setString(2, t.getCross_id()); preparedStatement.setString(2, t.getCross_id());
...@@ -788,12 +816,10 @@ public class TravelEventAndCycleCoProcessFunction extends CoProcessFunction<Trav ...@@ -788,12 +816,10 @@ public class TravelEventAndCycleCoProcessFunction extends CoProcessFunction<Trav
preparedStatement.setDouble(15, t.getAverage_stop_delay()); preparedStatement.setDouble(15, t.getAverage_stop_delay());
preparedStatement.setDouble(16, t.getAverage_stop_times()); preparedStatement.setDouble(16, t.getAverage_stop_times());
preparedStatement.setDouble(17, t.getAverage_car_head_time_gap()); preparedStatement.setDouble(17, t.getAverage_car_head_time_gap());
preparedStatement.addBatch(); preparedStatement.addBatch();
} }
preparedStatement.executeBatch(); preparedStatement.executeBatch();
// connection.close();
} }
......
realtime/hologram-streaming/src/main/java/com/zhht/irn/job/DirectionEvalJob.java
...@@ -6,23 +6,23 @@ import com.zhht.irn.entity.Cycle; ...@@ -6,23 +6,23 @@ import com.zhht.irn.entity.Cycle;
import com.zhht.irn.entity.TravelEvent; import com.zhht.irn.entity.TravelEvent;
import com.zhht.irn.entity.metric.DirectionEvalSecondMetric; import com.zhht.irn.entity.metric.DirectionEvalSecondMetric;
import com.zhht.irn.functions.CalDirectionSecondMetricFunction; import com.zhht.irn.functions.CalDirectionSecondMetricFunction;
import com.zhht.irn.sink.DirectionEvalSecondSink; import com.zhht.irn.utils.DateUtils;
import com.zhht.irn.source.CycleMockSource; import com.zhht.irn.utils.DorisUtils;
import com.zhht.irn.source.TravelEventMockSource;
import com.zhht.irn.utils.FlinkUtils; import com.zhht.irn.utils.FlinkUtils;
import com.zhht.irn.utils.StringUtils; import com.zhht.irn.utils.StringUtils;
import org.apache.doris.flink.sink.DorisSink;
import org.apache.flink.api.common.eventtime.WatermarkStrategy; import org.apache.flink.api.common.eventtime.WatermarkStrategy;
import org.apache.flink.api.common.functions.MapFunction;
import org.apache.flink.streaming.api.datastream.DataStream; import org.apache.flink.streaming.api.datastream.DataStream;
import org.apache.flink.streaming.api.datastream.KeyedStream; import org.apache.flink.streaming.api.datastream.KeyedStream;
import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment; import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment;
import org.apache.flink.streaming.api.functions.co.KeyedCoProcessFunction;
import org.apache.flink.streaming.api.windowing.time.Time;
import org.apache.flink.util.Collector;
import org.slf4j.Logger; import org.slf4j.Logger;
import org.slf4j.LoggerFactory; import org.slf4j.LoggerFactory;
import java.time.Duration; import java.time.Duration;
import static com.zhht.irn.utils.DateUtils.longToStandardFormatString;
/** /**
* 按方向评价实时任务，按秒计算 * 按方向评价实时任务，按秒计算
* *
...@@ -42,20 +42,18 @@ public class DirectionEvalJob { ...@@ -42,20 +42,18 @@ public class DirectionEvalJob {
// 获取信控数据-周期结束实时流 Kafka // 获取信控数据-周期结束实时流 Kafka
DataStream<Cycle> cycleStream = DataStream<Cycle> cycleStream =
FlinkUtils.createKafkaStream(args, env, Constant.CYCLE_TOPIC_NAME, "DirectionEvalJob") FlinkUtils.createKafkaStream(args, env, Constant.CYCLE_TOPIC_NAME, "DirectionEvalJob")
.map(record -> { .map(record -> {
System.out.println(record); return JSON.parseObject(record, Cycle.class);
return JSON.parseObject(record, Cycle.class); });
});
// 获取旅行事件数据实时流 kafka // 获取旅行事件数据实时流 kafka
DataStream<TravelEvent> travelEventStream = DataStream<TravelEvent> travelEventStream =
FlinkUtils.createKafkaStream(args, env, Constant.TRAVEL_EVENT_TOPIC_NAME, "DirectionEvalJob") FlinkUtils.createKafkaStream(args, env, Constant.TRAVEL_EVENT_TOPIC_NAME, "DirectionEvalJob")
.map(record -> { .map(record -> {
System.out.println(record); return JSON.parseObject(record, TravelEvent.class);
return JSON.parseObject(record, TravelEvent.class); }).filter(event -> {
}).filter(event -> { return StringUtils.isNotEmpty(event.getEventLineId());
return StringUtils.isNotEmpty(event.getEventLineId()); });
});
WatermarkStrategy<Cycle> cycleWatermarkStrategy = WatermarkStrategy WatermarkStrategy<Cycle> cycleWatermarkStrategy = WatermarkStrategy
.<Cycle>forBoundedOutOfOrderness((Duration.ofSeconds(3))) .<Cycle>forBoundedOutOfOrderness((Duration.ofSeconds(3)))
...@@ -71,12 +69,21 @@ public class DirectionEvalJob { ...@@ -71,12 +69,21 @@ public class DirectionEvalJob {
KeyedStream<TravelEvent, String> travelEventKeyedStream = travelEventStream.assignTimestampsAndWatermarks(travelEventWatermarkStrategy) KeyedStream<TravelEvent, String> travelEventKeyedStream = travelEventStream.assignTimestampsAndWatermarks(travelEventWatermarkStrategy)
.keyBy(record -> record.getCrossId()); .keyBy(record -> record.getCrossId());
// 定义好doris sink builder
DorisSink.Builder<String> builder = DorisUtils.getBuilder("doris-connector-direction-eval.properties");
// 周期数据join 旅行事件 // 周期数据join 旅行事件
cycleStream.assignTimestampsAndWatermarks(cycleWatermarkStrategy) cycleStream.assignTimestampsAndWatermarks(cycleWatermarkStrategy)
.keyBy(record -> record.getCrossCode()) .keyBy(record -> record.getCrossCode())
.connect(travelEventKeyedStream) .connect(travelEventKeyedStream)
.process(new CalDirectionSecondMetricFunction()) .process(new CalDirectionSecondMetricFunction())
.addSink(new DirectionEvalSecondSink()).setParallelism(4); .map((MapFunction<DirectionEvalSecondMetric, String>) desm -> {
// 将结果实体类，拼接成制表符分割的字符串，进行sink，要保持字段顺序跟表的字段顺序相一致
return desm.getCrossId() + "\t" + desm.getLineId() + "\t" + desm.getCycleOrder() + "\t" + longToStandardFormatString(desm.getCycleStart()) + "\t"
+ longToStandardFormatString(desm.getCycleEnd()) + "\t" + desm.getCurrentSecond() + "\t" + desm.getAccArrivedFlow() + "\t"
+ desm.getAccQueueLength() + "\t" + desm.getStopFlagCount() + "\t" + desm.getAccStopCount() + "\t"
+ longToStandardFormatString(System.currentTimeMillis()) + "\t" + longToStandardFormatString(System.currentTimeMillis());
}).sinkTo(builder.build());
// 旅行数据流 // 旅行数据流
env.execute("DirectionEvalJob"); env.execute("DirectionEvalJob");
......
realtime/hologram-streaming/src/main/java/com/zhht/irn/job/LaneNormJob.java
...@@ -11,6 +11,7 @@ import org.apache.flink.api.common.eventtime.WatermarkStrategy; ...@@ -11,6 +11,7 @@ import org.apache.flink.api.common.eventtime.WatermarkStrategy;
import org.apache.flink.api.common.functions.FilterFunction; import org.apache.flink.api.common.functions.FilterFunction;
import org.apache.flink.api.common.functions.MapFunction; import org.apache.flink.api.common.functions.MapFunction;
import org.apache.flink.api.common.restartstrategy.RestartStrategies; import org.apache.flink.api.common.restartstrategy.RestartStrategies;
import org.apache.flink.api.common.time.Time;
import org.apache.flink.api.java.functions.KeySelector; import org.apache.flink.api.java.functions.KeySelector;
import org.apache.flink.streaming.api.datastream.*; import org.apache.flink.streaming.api.datastream.*;
import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment; import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment;
...@@ -29,9 +30,11 @@ public class LaneNormJob { ...@@ -29,9 +30,11 @@ public class LaneNormJob {
public static void main(String[] args) throws Exception { public static void main(String[] args) throws Exception {
try { try {
StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment(); StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
env.setRestartStrategy(RestartStrategies.noRestart()); //env.setRestartStrategy(RestartStrategies.noRestart());
DataStream<String> cycleStringStream = FlinkUtils.createKafkaStream(args, env,"signal_cycle_data","laneNormJob"); //设置重启策略
DataStream<String> carStringStream = FlinkUtils.createKafkaStream(args, env,"trips_event_info","laneNormJob"); //env.setRestartStrategy( RestartStrategies.fixedDelayRestart(10, Time.seconds(10)));
DataStream<String> cycleStringStream = FlinkUtils.createKafkaStreamHuangzhun(args, env,"signal_cycle_data","laneNormJob1");
DataStream<String> carStringStream = FlinkUtils.createKafkaStreamHuangzhun(args, env,"trips_event_info","laneNormJob1");
KeySelector<Cycle, String> cycleKeySelect = new KeySelector<Cycle, String>() { KeySelector<Cycle, String> cycleKeySelect = new KeySelector<Cycle, String>() {
@Override @Override
public String getKey(Cycle cycle) throws Exception { public String getKey(Cycle cycle) throws Exception {
...@@ -79,7 +82,7 @@ public class LaneNormJob { ...@@ -79,7 +82,7 @@ public class LaneNormJob {
ConnectedStreams<Cycle, TravelEvent> cycleTravelEventConnectedStreams = cycleDataStreamSource.connect(carStream).keyBy(cycle -> cycle.getCrossCode(), travelEvent -> travelEvent.getCrossId()); ConnectedStreams<Cycle, TravelEvent> cycleTravelEventConnectedStreams = cycleDataStreamSource.connect(carStream).keyBy(cycle -> cycle.getCrossCode(), travelEvent -> travelEvent.getCrossId());
SingleOutputStreamOperator<List<LaneNorm>> process = cycleTravelEventConnectedStreams.process(new LaneNormProcessFunction()).setParallelism(4); SingleOutputStreamOperator<List<LaneNorm>> process = cycleTravelEventConnectedStreams.process(new LaneNormProcessFunction()).setParallelism(4);
process.addSink(new LaneNormSink()).name("laneNormJobSink"); process.addSink(new LaneNormSink()).name(" ");
env.execute("laneNormJob"); env.execute("laneNormJob");
}catch (Exception e){ }catch (Exception e){
System.out.println("程序异常关闭->"+e); System.out.println("程序异常关闭->"+e);
......
realtime/hologram-streaming/src/main/java/com/zhht/irn/job/TravelInfoJob.java
...@@ -5,8 +5,10 @@ import com.zhht.irn.consts.Constant; ...@@ -5,8 +5,10 @@ import com.zhht.irn.consts.Constant;
import com.zhht.irn.entity.Location; import com.zhht.irn.entity.Location;
import com.zhht.irn.entity.Travel; import com.zhht.irn.entity.Travel;
import com.zhht.irn.sink.TravelMetricSink; import com.zhht.irn.sink.TravelMetricSink;
import com.zhht.irn.utils.FileUtils; import com.zhht.irn.utils.DateUtils;
import com.zhht.irn.utils.DorisUtils;
import com.zhht.irn.utils.FlinkUtils; import com.zhht.irn.utils.FlinkUtils;
import org.apache.doris.flink.sink.DorisSink;
import org.apache.flink.api.common.functions.MapFunction; import org.apache.flink.api.common.functions.MapFunction;
import org.apache.flink.streaming.api.datastream.DataStream; import org.apache.flink.streaming.api.datastream.DataStream;
import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment; import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment;
...@@ -16,6 +18,7 @@ import org.slf4j.LoggerFactory; ...@@ -16,6 +18,7 @@ import org.slf4j.LoggerFactory;
import java.util.List; import java.util.List;
import static com.zhht.irn.utils.StringUtils.isNotEmpty; import static com.zhht.irn.utils.StringUtils.isNotEmpty;
import static com.zhht.irn.utils.DateUtils.longToStandardFormatString;
/** /**
* 旅行信息实时任务 * 旅行信息实时任务
...@@ -34,13 +37,18 @@ public class TravelInfoJob { ...@@ -34,13 +37,18 @@ public class TravelInfoJob {
DataStream<String> rawStringStream = FlinkUtils.createKafkaStream(args, env, Constant.TRAVEL_TOPIC_NAME, "TravelInfoJob"); DataStream<String> rawStringStream = FlinkUtils.createKafkaStream(args, env, Constant.TRAVEL_TOPIC_NAME, "TravelInfoJob");
// 接入数据的第一时间，要进行本地文件备份，以便后续问题排查，需要时打开 // 接入数据的第一时间，要进行本地文件备份，以便后续问题排查，需要时打开
// rawStringStream.sinkTo(FileUtils.getFileSink("./data/")); // rawStringStream.sinkTo(FileUtils.getFileSink("./data/
// 获取builder
DorisSink.Builder<String> builder = DorisUtils.getBuilder("doris-connector-travel-info.properties");
DataStream<Travel> travelStream = rawStringStream
// json 格式字符串转化为 Travel 实体类 @todo 不一定能正确解析
.map(record -> JSON.parseObject(record, Travel.class))
.filter(travel -> travel.getLocations() != null);
// 开始计算旅行信息相关指标 // 开始计算旅行信息相关指标
rawStringStream.map((MapFunction<String, Travel>) s -> { travelStream.map(travel -> {
// json 格式字符串转化为 Travel 实体类 @todo 不一定能正确解析 // -1 代表该值从未被处理过
Travel travel = JSON.parseObject(s, Travel.class);
// -1 代表该值从未被处理过
double minSpeed = -1, maxSpeed = -1, totalSpeed = -1; double minSpeed = -1, maxSpeed = -1, totalSpeed = -1;
int accTravel = 0;// 拥有完整开始、结束时刻的旅行记录数 int accTravel = 0;// 拥有完整开始、结束时刻的旅行记录数
...@@ -85,7 +93,7 @@ public class TravelInfoJob { ...@@ -85,7 +93,7 @@ public class TravelInfoJob {
boolean stopped = false; boolean stopped = false;
List<Location> locations = travel.getLocations(); List<Location> locations = travel.getLocations();
long stopTime = 0; int stopTime = 0;
int stopCount = 0; int stopCount = 0;
for(int j = 0; j < locations.size(); j ++) { for(int j = 0; j < locations.size(); j ++) {
...@@ -123,7 +131,19 @@ public class TravelInfoJob { ...@@ -123,7 +131,19 @@ public class TravelInfoJob {
"停车总时长：{}", travel.getRecordId(), maxSpeed, minSpeed, travel.getAvgSpeed(), stopCount, stopTime); "停车总时长：{}", travel.getRecordId(), maxSpeed, minSpeed, travel.getAvgSpeed(), stopCount, stopTime);
return travel; return travel;
}).addSink(new TravelMetricSink()); }).map((MapFunction<Travel, String>) travel -> {
return travel.getRecordId() + "\t" + travel.getCrossId() + "\t" + longToStandardFormatString(travel.getRecordTime()) + "\t"
+ longToStandardFormatString(travel.getTravelBeginTime()) + "\t" + longToStandardFormatString(travel.getTravelEndTime()) + "\t"
+ travel.getFlowDirection() + "\t" + travel.getCarId() + "\t" + travel.getCarColor() + "\t" + travel.getPlate() + "\t"
+ travel.getPlateColor() + "\t" + travel.getCarType() + "\t" + travel.getFirstLineId() + "\t" + longToStandardFormatString(travel.getFirstTime()) + "\t"
+ travel.getFirstSpeed() + "\t" + travel.getArrivedLineId() + "\t" + longToStandardFormatString(travel.getArrivedTime()) + "\t"
+ travel.getArrivedSpeed() + "\t" + travel.getInCrossLineId() + "\t" + longToStandardFormatString(travel.getInCrossTime()) + "\t"
+ travel.getInSpeed() + "\t" + travel.getOutCrossLineId() + "\t" + longToStandardFormatString(travel.getOutCrossTime()) + "\t"
+ travel.getOutSpeed() + "\t" + travel.getAwayLineId() + "\t" + longToStandardFormatString(travel.getAwayTime()) + "\t"
+ travel.getAwaySpeed() + "\t" + travel.getLastLineId() + "\t" + longToStandardFormatString(travel.getLastTime()) + "\t"
+ travel.getLastSpeed() + "\t" + travel.getCrossingTime() + "\t" + travel.getMinSpeed() + "\t" + travel.getMaxSpeed() + "\t"
+ travel.getAvgSpeed() + "\t" + travel.getStopCount() + "\t" + travel.getStopTime() + "\t" + longToStandardFormatString(System.currentTimeMillis());
}).sinkTo(builder.build());
env.execute("TravelInfoJob"); env.execute("TravelInfoJob");
} }
......
realtime/hologram-streaming/src/main/java/com/zhht/irn/job/TravelSituationAnalysisJob.java
...@@ -2,21 +2,33 @@ package com.zhht.irn.job; ...@@ -2,21 +2,33 @@ package com.zhht.irn.job;
import com.zhht.irn.entity.dto.CycleSignalData; import com.zhht.irn.entity.dto.CycleSignalData;
import com.zhht.irn.entity.dto.TravelEvent; import com.zhht.irn.entity.dto.TravelEvent;
import com.zhht.irn.entity.dto.TravelInfo;
import com.zhht.irn.entity.dto.TravelLineSinkInfo;
import com.zhht.irn.functions.TravelCarInfoCoProcessFunction;
import com.zhht.irn.functions.TravelEventAndCycleCoProcessFunction; import com.zhht.irn.functions.TravelEventAndCycleCoProcessFunction;
import com.zhht.irn.schema.CycleSignalKafkaSchema; import com.zhht.irn.schema.CycleSignalKafkaSchema;
import com.zhht.irn.schema.TravelEventKafkaSchema; import com.zhht.irn.schema.TravelEventKafkaSchema;
import com.zhht.irn.schema.TravelInfoKafkaSchema;
import com.zhht.irn.sink.TravelLaneInfoSink;
import com.zhht.irn.utils.DorisUtils;
import org.apache.flink.api.common.restartstrategy.RestartStrategies; import org.apache.flink.api.common.restartstrategy.RestartStrategies;
import org.apache.flink.api.common.serialization.SimpleStringSchema;
import org.apache.flink.api.common.time.Time; import org.apache.flink.api.common.time.Time;
import org.apache.flink.api.java.functions.KeySelector; import org.apache.flink.api.java.functions.KeySelector;
import org.apache.flink.api.java.utils.ParameterTool;
import org.apache.flink.runtime.state.filesystem.FsStateBackend;
import org.apache.flink.streaming.api.CheckpointingMode; import org.apache.flink.streaming.api.CheckpointingMode;
import org.apache.flink.streaming.api.datastream.ConnectedStreams; import org.apache.flink.streaming.api.datastream.ConnectedStreams;
import org.apache.flink.streaming.api.datastream.DataStreamSource; import org.apache.flink.streaming.api.datastream.DataStreamSource;
import org.apache.flink.streaming.api.datastream.SingleOutputStreamOperator;
import org.apache.flink.streaming.api.environment.CheckpointConfig; import org.apache.flink.streaming.api.environment.CheckpointConfig;
import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment; import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment;
import org.apache.flink.streaming.connectors.kafka.FlinkKafkaConsumer; import org.apache.flink.streaming.connectors.kafka.FlinkKafkaConsumer;
import org.apache.flink.streaming.connectors.kafka.FlinkKafkaConsumerBase;
import org.apache.kafka.clients.consumer.ConsumerConfig; import org.apache.kafka.clients.consumer.ConsumerConfig;
import java.io.InputStream;
import java.util.List;
import java.util.Properties; import java.util.Properties;
/** /**
...@@ -26,85 +38,88 @@ import java.util.Properties; ...@@ -26,85 +38,88 @@ import java.util.Properties;
* @create 2022-11-14 11:43 * @create 2022-11-14 11:43
**/ **/
public class TravelSituationAnalysisJob { public class TravelSituationAnalysisJob {
public static void main(String[] args) { public static void main(String[] args) {
try { try {
InputStream inputStream = DorisUtils.class.getClassLoader().getResourceAsStream("kafka.properties");
ParameterTool tool = ParameterTool.fromPropertiesFile(inputStream);
//1、创建环境 //1、创建环境
StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment(); StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
// checkpoint配置 // checkpoint配置
CheckpointConfig checkpointConfig = env.getCheckpointConfig(); CheckpointConfig checkpointConfig = env.getCheckpointConfig();
// checkpoint时间间隔3分钟 // checkpoint时间间隔3分钟
checkpointConfig.setCheckpointInterval(1* 60 * 1000); checkpointConfig.setCheckpointInterval( 60 * 1000);
// 两次checkpoint中最短时间间隔1分钟 // 两次checkpoint中最短时间间隔1分钟
checkpointConfig.setMinPauseBetweenCheckpoints(60 * 1000); checkpointConfig.setMinPauseBetweenCheckpoints(60 * 1000);
// 同时同时允许1个checkpoint进行 // 同时同时允许1个checkpoint进行
checkpointConfig.setMaxConcurrentCheckpoints(1); checkpointConfig.setMaxConcurrentCheckpoints(1);
//这如果不指定使用FsStateBackend 则默认使用flink集群中的配置，rocksDB
// env.setStateBackend(new FsStateBackend("file:///data/flink/state"));
env.setStateBackend(new FsStateBackend(tool.getRequired("checkpoint.path")));
env.getCheckpointConfig().setCheckpointingMode(CheckpointingMode.EXACTLY_ONCE); env.getCheckpointConfig().setCheckpointingMode(CheckpointingMode.EXACTLY_ONCE);
env.getCheckpointConfig().setExternalizedCheckpointCleanup(CheckpointConfig.ExternalizedCheckpointCleanup.RETAIN_ON_CANCELLATION);
//设置重启策略 //设置重启策略 这里不设置重启策略，由平台统一拉起任务
env.setRestartStrategy( RestartStrategies.fixedDelayRestart(5, Time.seconds(10))); // env.setRestartStrategy(RestartStrategies.fixedDelayRestart(10, Time.seconds(30)));
env.setRestartStrategy(RestartStrategies.noRestart());
/*
关于flink checkpoint存储的设置统一在集群配置文件中设置
checkpointConfig.enableExternalizedCheckpoints(CheckpointConfig.ExternalizedCheckpointCleanup.RETAIN_ON_CANCELLATION);
env.setStateBackend(new RocksDBStateBackend("file:///opt/rocksDb/", true));*/
Properties kafkaProperties = new Properties(); Properties kafkaProperties = new Properties();
kafkaProperties.setProperty("bootstrap.servers", "dn1.zhht:9092,dn2.zhht:9092,dn3.zhht:9092"); // kafkaProperties.setProperty("bootstrap.servers", "dn3.zhht:9092");
// kafkaProperties.setProperty("bootstrap.servers", "172.25.1.251:9092,172.25.1.122:9092,172.25.1.67:9092"); kafkaProperties.setProperty("bootstrap.servers", tool.getRequired("bootstrap.servers"));
kafkaProperties.setProperty("group.id", "travel_event"); // kafkaProperties.setProperty("bootstrap.servers", "139.9.157.176:9092");
// kafkaProperties.setProperty("group.id", "travel_event");
kafkaProperties.setProperty("group.id", "trips_info2");
kafkaProperties.setProperty(ConsumerConfig.ENABLE_AUTO_COMMIT_CONFIG, "true"); kafkaProperties.setProperty(ConsumerConfig.ENABLE_AUTO_COMMIT_CONFIG, "true");
kafkaProperties.setProperty(ConsumerConfig.AUTO_OFFSET_RESET_CONFIG, "latest"); kafkaProperties.setProperty(ConsumerConfig.AUTO_OFFSET_RESET_CONFIG, "latest");
Properties kafkaProperties2 = new Properties(); Properties kafkaProperties2 = new Properties();
kafkaProperties2.setProperty("bootstrap.servers", "dn1.zhht:9092,dn2.zhht:9092,dn3.zhht:9092"); // kafkaProperties2.setProperty("bootstrap.servers", "dn3.zhht:9092");
// kafkaProperties2.setProperty("bootstrap.servers", "172.25.1.251:9092,172.25.1.122:9092,172.25.1.67:9092"); kafkaProperties2.setProperty("bootstrap.servers", tool.getRequired("bootstrap.servers"));
kafkaProperties2.setProperty("group.id", "cycle_signal"); // kafkaProperties2.setProperty("bootstrap.servers", "139.9.157.176:9092");
kafkaProperties2.setProperty("group.id", "cycle_signal2");
kafkaProperties2.setProperty(ConsumerConfig.ENABLE_AUTO_COMMIT_CONFIG, "true"); kafkaProperties2.setProperty(ConsumerConfig.ENABLE_AUTO_COMMIT_CONFIG, "true");
kafkaProperties2.setProperty(ConsumerConfig.AUTO_OFFSET_RESET_CONFIG, "latest"); kafkaProperties2.setProperty(ConsumerConfig.AUTO_OFFSET_RESET_CONFIG, "latest");
//旅行事件
FlinkKafkaConsumerBase<TravelEvent> travelEvent =
new FlinkKafkaConsumer<TravelEvent>("trips_event_info",
new TravelEventKafkaSchema(),
kafkaProperties) {
};
DataStreamSource<TravelEvent> travelEventStream = env.addSource(travelEvent);
travelEventStream.print(); //旅行信息数据
// FlinkKafkaConsumer<TravelInfo> travelInfo = new FlinkKafkaConsumer<>("t_info2", new TravelInfoKafkaSchema(), kafkaProperties);
FlinkKafkaConsumer<TravelInfo> travelInfo = new FlinkKafkaConsumer<>("trips_info", new TravelInfoKafkaSchema(), kafkaProperties);
DataStreamSource<TravelInfo> travelInfoStream = env.addSource(travelInfo).setParallelism(3);
//周期信号 //周期信号
FlinkKafkaConsumer<CycleSignalData> cycleSignalData = // FlinkKafkaConsumer<CycleSignalData> cycleSignalData = new FlinkKafkaConsumer<>("s_data2", new CycleSignalKafkaSchema(), kafkaProperties2);
new FlinkKafkaConsumer<>("signal_cycle_data", FlinkKafkaConsumer<CycleSignalData> cycleSignalData = new FlinkKafkaConsumer<>("signal_cycle_data", new CycleSignalKafkaSchema(), kafkaProperties2);
new CycleSignalKafkaSchema(),
kafkaProperties2);
DataStreamSource<CycleSignalData> cycleSignalDataStream = env.addSource(cycleSignalData); DataStreamSource<CycleSignalData> cycleSignalDataStream = env.addSource(cycleSignalData);
cycleSignalDataStream.print();
//2、使用connect，把2个流联合处理 //2、使用connect，把2个流联合处理
ConnectedStreams<TravelEvent, CycleSignalData> connect = travelEventStream ConnectedStreams<TravelInfo, CycleSignalData> connect = travelInfoStream
.connect(cycleSignalDataStream); .connect(cycleSignalDataStream);
ConnectedStreams<TravelInfo, CycleSignalData> travelInfoCycleSignalDataConnectedStreams = connect.keyBy(
(KeySelector<TravelInfo, String>) travelInfo1 -> travelInfo1.getCrossId(),
(KeySelector<CycleSignalData, String>) cycleSignalData1 -> cycleSignalData1.getCrossCode()
);
ConnectedStreams<TravelEvent, CycleSignalData> travelInfoCycleSignalDataConnectedStreams = connect.keyBy( SingleOutputStreamOperator<List<TravelLineSinkInfo>> sinkDataStream = travelInfoCycleSignalDataConnectedStreams
(KeySelector<TravelEvent, String>) travelInfo1 -> travelInfo1.getCrossId(), .process(new TravelCarInfoCoProcessFunction()).setParallelism(3);
(KeySelector<CycleSignalData, String>) cycleSignalData1 -> cycleSignalData1.getCrossCode()
);
travelInfoCycleSignalDataConnectedStreams.process(new TravelEventAndCycleCoProcessFunction()); sinkDataStream.addSink(new TravelLaneInfoSink()).setParallelism(3);
env.execute(); env.execute("TravelSituationAnalysisJob");
} catch (Exception e){ } catch (Exception e){
e.printStackTrace(); e.printStackTrace();
} }
} }
} }
realtime/hologram-streaming/src/main/java/com/zhht/irn/sink/DirectionEvalSecondSink.java deleted 100644 → 0
package com.zhht.irn.sink;
import com.zhht.irn.entity.metric.DirectionEvalSecondMetric;
import com.zhht.irn.utils.DateUtils;
import com.zhht.irn.utils.DorisUtils;
import com.zhht.irn.utils.DruidConnectPoolUtils;
import com.zhht.irn.utils.MySQLUtils;
import org.apache.flink.configuration.Configuration;
import org.apache.flink.streaming.api.functions.sink.RichSinkFunction;
import java.sql.Connection;
import java.sql.Date;
import java.sql.PreparedStatement;
import static com.zhht.irn.consts.Constant.INSERT_BATCH_SIZE;
/**
* 按方向评价实时指标
*
* 结果表：app_direction_eval_second_metric
* 计算结果指标包含 交通流量，排队长度，延误时间，停车次数
*/
public class DirectionEvalSecondSink extends RichSinkFunction<DirectionEvalSecondMetric> {
Connection connection;
PreparedStatement insertPstmt;
int counter = 0;
@Override
public void open(Configuration parameters) throws Exception {
super.open(parameters);
connection = DorisUtils.getConnection();
String insertSql="insert into app_direction_eval_second_metric" +
"(cross_id, lane_id, cycle_order, current_second, acc_arrived_flow, acc_queue_length, acc_stop_count," +
" stop_flag_count, statistic_time, cycle_start_time, cycle_end_time, record_date) values (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)";
insertPstmt = connection.prepareStatement(insertSql);
}
@Override
public void close() throws Exception {
super.close();
if (insertPstmt != null) insertPstmt.close();
if (connection != null) connection.close();
}
@Override
public void invoke(DirectionEvalSecondMetric metric, Context context) throws Exception {
insertPstmt.setString(1, metric.getCrossId());
insertPstmt.setString(2, metric.getLineId());
insertPstmt.setLong(3, metric.getCycleOrder());
insertPstmt.setLong(4, metric.getCurrentSecond());
insertPstmt.setInt(5, metric.getAccArrivedFlow());
insertPstmt.setInt(6, metric.getAccQueueLength());
insertPstmt.setInt(7, metric.getAccStopCount());
insertPstmt.setInt(8, metric.getStopFlagCount());
insertPstmt.setString(9, DateUtils.getTodayTime());
insertPstmt.setString(10, DateUtils.longToString(metric.getCycleStart(), DateUtils.timeFormat));
insertPstmt.setString(11, DateUtils.longToString(metric.getCycleEnd(), DateUtils.timeFormat));
insertPstmt.setDate(12, new Date(System.currentTimeMillis()));
if(counter < INSERT_BATCH_SIZE) {
counter ++;
insertPstmt.addBatch();
} else {
insertPstmt.executeBatch();
counter = 0;
}
}
}
realtime/hologram-streaming/src/main/java/com/zhht/irn/sink/LaneNormSink.java
...@@ -52,6 +52,7 @@ public class LaneNormSink extends RichSinkFunction<List<LaneNorm>> { ...@@ -52,6 +52,7 @@ public class LaneNormSink extends RichSinkFunction<List<LaneNorm>> {
*/ */
@Override @Override
public void invoke(List<LaneNorm> laneNorms, SinkFunction.Context context) throws Exception { public void invoke(List<LaneNorm> laneNorms, SinkFunction.Context context) throws Exception {
if(laneNorms.size()==0){ if(laneNorms.size()==0){
return; return;
} }
......
realtime/hologram-streaming/src/main/java/com/zhht/irn/sink/TravelLaneInfoSink.java 0 → 100644
package com.zhht.irn.sink;
import com.alibaba.druid.pool.DruidPooledConnection;
import com.zhht.irn.entity.dto.TravelLineSinkInfo;
import com.zhht.irn.functions.TravelCarInfoCoProcessFunction;
import com.zhht.irn.utils.DorisUtils;
import com.zhht.irn.utils.DruidConnectPoolUtils;
import org.apache.flink.configuration.Configuration;
import org.apache.flink.streaming.api.functions.sink.RichSinkFunction;
import org.apache.log4j.Logger;
import java.sql.Connection;
import java.sql.PreparedStatement;
import java.text.SimpleDateFormat;
import java.util.Date;
import java.util.List;
/**
* TravelLaneInfoSink
*
* @author Rui meng
* @description 行车情况信息sink
* @date 2022/12/15 9:39
*/
public class TravelLaneInfoSink extends RichSinkFunction<List<TravelLineSinkInfo>> {
private static Logger logger = Logger.getLogger(TravelLaneInfoSink.class);
Connection connection;
@Override
public void open(Configuration parameters) throws Exception {
super.open(parameters);
connection = DorisUtils.getConnection();
logger.info("建立数据库连接成功。。。。。。。");
}
@Override
public void close() throws Exception {
super.close();
if(connection!=null){
connection.close();
}
}
@Override
public void invoke(List<TravelLineSinkInfo> value, Context context) throws Exception {
super.invoke(value, context);
//重新获取连接，避免超时的问题
connection = DorisUtils.getConnection();
String sql = "INSERT INTO app_cross_line_travel_car_info" +
"(record_date,cross_id,statistic_time,cycle_begin_time,cycle_end_time, lane_id, cycle_order, direction," +
" flow_direction, pass_numbers," +
" last_car_inCross_time, average_pass_time, average_pass_speed, average_control_delay, " +
"average_stop_delay, average_stop_times, average_car_head_time_gap" +
")" +
"VALUES(?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?,?,?,?,?) ";
PreparedStatement preparedStatement = connection.prepareStatement(sql);
//TODO 这里的时间使用数据携带的时间，使用最后进入路口车辆的时间，这样在补数据时，可以补充到对应的分区下
Date date = new Date();
SimpleDateFormat sdf = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss");
SimpleDateFormat sdf2 = new SimpleDateFormat("yyyy-MM-dd");
String statistic_time = sdf.format(date);
for (TravelLineSinkInfo t :value) {
// 使用事件数据的时间划分数据分区，这样可以进行补数操作
String record_date = sdf2.format(t.getLast_car_inCross_time());
preparedStatement.setString(1, record_date);
preparedStatement.setString(2, t.getCross_id());
preparedStatement.setString(3, statistic_time);
preparedStatement.setString(4, sdf.format(t.getCycle_begin_time()));
preparedStatement.setString(5, sdf.format(t.getCycle_end_time()));
preparedStatement.setString(6, t.getLane_id());
preparedStatement.setString(7, t.getCycle_id().toString());
preparedStatement.setString(8, t.getDirection());
preparedStatement.setString(9, t.getFlow_direction());
preparedStatement.setInt(10, t.getPass_numbers());
preparedStatement.setLong(11, t.getLast_car_inCross_time());
preparedStatement.setDouble(12, t.getAverage_pass_time());
preparedStatement.setDouble(13, t.getAverage_pass_speed());
preparedStatement.setDouble(14, t.getAverage_control_delay());
preparedStatement.setDouble(15, t.getAverage_stop_delay());
preparedStatement.setDouble(16, t.getAverage_stop_times());
preparedStatement.setDouble(17, t.getAverage_car_head_time_gap());
preparedStatement.addBatch();
}
int[] ints = preparedStatement.executeBatch();
logger.info("写入数据成功。。。。。"+ints.length+"...");
preparedStatement.close();
connection.close();
}
}
realtime/hologram-streaming/src/main/java/com/zhht/irn/utils/DateUtils.java
...@@ -1038,6 +1038,15 @@ public class DateUtils { ...@@ -1038,6 +1038,15 @@ public class DateUtils {
} catch(Exception e) { e.printStackTrace();} } catch(Exception e) { e.printStackTrace();}
return longToString(time, dateFormat); return longToString(time, dateFormat);
} }
public static String longToStandardFormatString(String timestamp) {
return longToString(timestamp, "yyyy-MM-dd HH:mm:ss");
}
public static String longToStandardFormatString(long timestamp) {
return longToString(timestamp, "yyyy-MM-dd HH:mm:ss");
}
/** /**
* 将字符串日期转换为日期格式 * 将字符串日期转换为日期格式
* 自定義格式 * 自定義格式
......
realtime/hologram-streaming/src/main/java/com/zhht/irn/utils/DorisUtils.java
package com.zhht.irn.utils; package com.zhht.irn.utils;
import org.apache.doris.flink.cfg.DorisExecutionOptions;
import org.apache.doris.flink.cfg.DorisOptions;
import org.apache.doris.flink.cfg.DorisReadOptions;
import org.apache.doris.flink.sink.DorisSink;
import org.apache.doris.flink.sink.writer.SimpleStringSerializer;
import org.apache.flink.api.java.utils.ParameterTool; import org.apache.flink.api.java.utils.ParameterTool;
import java.io.IOException;
import java.io.InputStream; import java.io.InputStream;
import java.sql.Connection; import java.sql.Connection;
import java.sql.DriverManager; import java.sql.DriverManager;
...@@ -44,4 +50,34 @@ public class DorisUtils { ...@@ -44,4 +50,34 @@ public class DorisUtils {
} }
} }
} }
public static DorisSink.Builder<String> getBuilder(String filePath) throws IOException {
DorisSink.Builder<String> builder = DorisSink.builder();
DorisOptions.Builder dorisBuilder = DorisOptions.builder();
InputStream inputStream = DorisUtils.class.getClassLoader().getResourceAsStream(filePath);
ParameterTool tool = ParameterTool.fromPropertiesFile(inputStream);
// 根据路径的配置文件获取MySQL链接参数
String username = tool.getRequired("username");
String password = tool.getRequired("password");
String host = tool.getRequired("host");
String port = tool.getRequired("port");
String database = tool.get("database", "default");
String table = tool.getRequired("table");
String label = tool.getRequired("label");
dorisBuilder.setFenodes(host + ":" + port)
.setTableIdentifier(database + "." + table)
.setUsername(username)
.setPassword(password);
DorisExecutionOptions.Builder executionBuilder = DorisExecutionOptions.builder();
executionBuilder.setLabelPrefix(label); // streamload label prefix
builder.setDorisReadOptions(DorisReadOptions.builder().build())
.setDorisExecutionOptions(executionBuilder.build())
.setSerializer(new SimpleStringSerializer()) //serialize according to string
.setDorisOptions(dorisBuilder.build());
return builder;
}
} }
realtime/hologram-streaming/src/main/java/com/zhht/irn/utils/DruidConnectPoolUtils.java
...@@ -31,8 +31,8 @@ public class DruidConnectPoolUtils { ...@@ -31,8 +31,8 @@ public class DruidConnectPoolUtils {
dataSource.setUrl(dbHosts); dataSource.setUrl(dbHosts);
dataSource.setUsername(username); dataSource.setUsername(username);
dataSource.setPassword(password); dataSource.setPassword(password);
dataSource.setInitialSize(3); //初始连接数，默认0 dataSource.setInitialSize(5); //初始连接数，默认0
dataSource.setMaxActive(10); //最大连接数，默认8 dataSource.setMaxActive(30); //最大连接数，默认8
dataSource.setMinIdle(5); //最小闲置数 dataSource.setMinIdle(5); //最小闲置数
dataSource.setMaxWait(3000);//获取连接的最大等待时间，单位毫秒 dataSource.setMaxWait(3000);//获取连接的最大等待时间，单位毫秒
dataSource.setTestWhileIdle(true); //指明连接是否被空闲连接回收器(如果有)进行检验.如果检测失败,则连接将被从池中去除 dataSource.setTestWhileIdle(true); //指明连接是否被空闲连接回收器(如果有)进行检验.如果检测失败,则连接将被从池中去除
...@@ -47,8 +47,8 @@ public class DruidConnectPoolUtils { ...@@ -47,8 +47,8 @@ public class DruidConnectPoolUtils {
dataSource.setUrl(dbHosts); dataSource.setUrl(dbHosts);
dataSource.setUsername(username); dataSource.setUsername(username);
dataSource.setPassword(password); dataSource.setPassword(password);
dataSource.setInitialSize(3); //初始连接数，默认0 dataSource.setInitialSize(5); //初始连接数，默认0
dataSource.setMaxActive(10); //最大连接数，默认8 dataSource.setMaxActive(30); //最大连接数，默认8
dataSource.setMinIdle(5); //最小闲置数 dataSource.setMinIdle(5); //最小闲置数
dataSource.setMaxWait(3000);//获取连接的最大等待时间，单位毫秒 dataSource.setMaxWait(3000);//获取连接的最大等待时间，单位毫秒
dataSource.setTestWhileIdle(true); //指明连接是否被空闲连接回收器(如果有)进行检验.如果检测失败,则连接将被从池中去除 dataSource.setTestWhileIdle(true); //指明连接是否被空闲连接回收器(如果有)进行检验.如果检测失败,则连接将被从池中去除
......
realtime/hologram-streaming/src/main/java/com/zhht/irn/utils/FlinkUtils.java
...@@ -27,7 +27,7 @@ public class FlinkUtils { ...@@ -27,7 +27,7 @@ public class FlinkUtils {
private static final Logger logger = LoggerFactory.getLogger(FlinkUtils.class); private static final Logger logger = LoggerFactory.getLogger(FlinkUtils.class);
public static StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment(); public static StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
private static String defaultBootstrapServers; private static String defaultBootstrapServers;
private static String defaultInputTopics; private static String defaultInputTopics;
...@@ -152,10 +152,45 @@ public class FlinkUtils { ...@@ -152,10 +152,45 @@ public class FlinkUtils {
env.enableCheckpointing(checkpointInterval); env.enableCheckpointing(checkpointInterval);
env.setStateBackend(new FsStateBackend(checkpointPath)); env.setStateBackend(new FsStateBackend(checkpointPath));
env.getCheckpointConfig().enableExternalizedCheckpoints(CheckpointConfig.ExternalizedCheckpointCleanup.RETAIN_ON_CANCELLATION); env.getCheckpointConfig().enableExternalizedCheckpoints(CheckpointConfig.ExternalizedCheckpointCleanup.RETAIN_ON_CANCELLATION);
env.setRestartStrategy(RestartStrategies.fixedDelayRestart(2, Time.of(5, TimeUnit.SECONDS))); /**
* 暂时不需要失败自动重试
*/
//env.setRestartStrategy(RestartStrategies.fixedDelayRestart(2, Time.of(5, TimeUnit.SECONDS)));
// 基于Kafka 原始JSON格式的字符串，获取消息流
FlinkKafkaConsumer<String> kafkaConsumer = new FlinkKafkaConsumer<>(topicList, new SimpleStringSchema(), properties);
DataStream<String> rawStringStream = env.addSource(kafkaConsumer);
return rawStringStream;
}
public static DataStream<String> createKafkaStreamHuangzhun(String[] args, StreamExecutionEnvironment environment, String topics, String groupId) {
ParameterTool kafkaTool = ParameterTool.fromArgs(args);
String servers = kafkaTool.get("bootstrap.servers", defaultBootstrapServers);
String topics2 = kafkaTool.get("kafka.input.topics", topics);
String groupId2 = kafkaTool.get("group.id", groupId);
String autoOffsetReset = kafkaTool.get("auto.offset.reset", defaultAutoOffsetReset);
String checkpointPath = kafkaTool.get("checkpoint.path", defaultCheckpointPath);
long checkpointInterval = kafkaTool.getLong("checkpoint.interval", defaultCheckpointInterval);
String enableAutoCommit = kafkaTool.get("enable.auto.commit", "false");
return buildKafkaDataStreamHuangzhun(environment, servers, topics2, groupId2, autoOffsetReset, checkpointPath, checkpointInterval, enableAutoCommit);
}
private static DataStream<String> buildKafkaDataStreamHuangzhun(StreamExecutionEnvironment env, String servers, String topics, String groupId,
String autoOffsetReset, String checkpointPath, long checkpointInterval, String enableAutoCommit) {
List<String> topicList = Arrays.asList(topics.split(","));
Properties properties = new Properties();
properties.setProperty("bootstrap.servers", servers);
properties.setProperty("group.id", groupId);
properties.setProperty("enable.auto.commit", enableAutoCommit);
properties.setProperty("auto.offset.reset", autoOffsetReset);
env.enableCheckpointing(checkpointInterval);
env.setStateBackend(new FsStateBackend(checkpointPath));
env.getCheckpointConfig().enableExternalizedCheckpoints(CheckpointConfig.ExternalizedCheckpointCleanup.RETAIN_ON_CANCELLATION);
//env.setRestartStrategy(RestartStrategies.fixedDelayRestart(2, Time.of(5, TimeUnit.SECONDS)));
// 基于Kafka 原始JSON格式的字符串，获取消息流 // 基于Kafka 原始JSON格式的字符串，获取消息流
FlinkKafkaConsumer<String> kafkaConsumer = new FlinkKafkaConsumer<>(topicList, new SimpleStringSchema(), properties); FlinkKafkaConsumer<String> kafkaConsumer = new FlinkKafkaConsumer<>(topicList, new SimpleStringSchema(), properties);
kafkaConsumer.setStartFromLatest();
DataStream<String> rawStringStream = env.addSource(kafkaConsumer); DataStream<String> rawStringStream = env.addSource(kafkaConsumer);
return rawStringStream; return rawStringStream;
} }
......
realtime/hologram-streaming/src/main/java/com/zhht/irn/utils/applicationUtils.java 0 → 100644
package com.zhht.irn.utils;
import org.apache.flink.api.java.utils.ParameterTool;
import java.io.InputStream;
import java.sql.Connection;
import java.sql.DriverManager;
/**
* @Description:
* @Author ：Marinh
* @Param:
* @retrun:
* @Creat :2023-01-06-12:32
**/
public class applicationUtils {
public static String getLoggerLevel() throws Exception {
InputStream inputStream = DorisUtils.class.getClassLoader().getResourceAsStream("application.properties");
ParameterTool tool = ParameterTool.fromPropertiesFile(inputStream);
// 根据路径的配置文件获取MySQL链接参数
String driver = tool.getRequired("log.info.level");
return driver;
}
}
realtime/hologram-streaming/src/main/resources/doris-connector-travel-info.properties 0 → 100644
host=10.243.0.22
port=8030
username=root
password=mima
database=qxlk
table=app_travel_metric
label=app_travel_metric-import
\ No newline at end of file
realtime/hologram-streaming/src/main/resources/doris-connector.properties 0 → 100644
host=10.243.0.22
port=8030
username=root
password=mima
database=qxlk
table=app_direction_eval_second_metric
label=app_direction_eval_second_metric-import
\ No newline at end of file
realtime/hologram-streaming/src/main/resources/doris.properties
driver=com.mysql.jdbc.Driver driver=com.mysql.jdbc.Driver
host=10.243.0.22 host=localhost
port=9030 port=9030
username=root username=root
password=mima password=mima
......
realtime/hologram-streaming/src/main/resources/kafka.properties
...@@ -4,4 +4,4 @@ auto.offset.reset=latest ...@@ -4,4 +4,4 @@ auto.offset.reset=latest
group.id=TravelInfoJob group.id=TravelInfoJob
checkpoint.path=file:///data/flink/state checkpoint.path=file:///data/flink/state
checkpoint.interval=60000 checkpoint.interval=60000
enable.auto.commit=false enable.auto.commit=true
\ No newline at end of file \ No newline at end of file
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment