Skip to content

Z
ZHHT-IRN-BD-ANALYSIS
Commit c016f8ef by 吴延飞
Options

Merge remote-tracking branch 'origin/master'

parents 0867941d 2930fc23
Showing with 86 additions and 13 deletions
realtime/hologram-streaming/src/main/java/com/zhht/irn/functions/TravelEventAndCycleCoProcessFunction.java
......@@ -87,16 +87,16 @@ public class TravelEventAndCycleCoProcessFunction extends CoProcessFunction<Trav
carRecordState = getRuntimeContext().getMapState(carRecordStateDescriptor);
dataSource = DruidConnectPoolUtils.getDataSource
connection = DruidConnectPoolUtils.getConnection();
// ("jdbc:mysql://localhost:9030/demo?characterEncoding=utf8",
// ("jdbc:mysql://localhost:3307/test?characterEncoding=utf8",
// ("jdbc:mysql://10.243.0.26:3306/test?characterEncoding=utf8",
("jdbc:mysql://10.243.0.22:9030/demo?characterEncoding=utf8",
/* ("jdbc:mysql://10.243.0.22:9030/demo?characterEncoding=utf8",
"root",
// "root123");
"mima");
connection = dataSource.getConnection();
*/
// dataSource.getConnection();
dim_cnt_cross_lane_position = getLaneDimData(connection);
}
......@@ -754,7 +754,7 @@ public class TravelEventAndCycleCoProcessFunction extends CoProcessFunction<Trav
*/
private void sinkTravelLineSinkInfoToMysql(List<TravelLineSinkInfo> list) throws SQLException {
DruidPooledConnection connection = dataSource.getConnection();
DruidPooledConnection connection = DruidConnectPoolUtils.getConnection();
String sql = "INSERT INTO demo.app_cross_line_travel_car_info" +
"(record_date,cross_id,statistic_time,cycle_begin_time,cycle_end_time, lane_id, cycle_order, direction," +
......
realtime/hologram-streaming/src/main/java/com/zhht/irn/job/LaneNormJob.java
......@@ -5,6 +5,7 @@ import com.zhht.irn.entity.*;
import com.zhht.irn.enums.EventType;
import com.zhht.irn.functions.LaneNormProcessFunction;
import com.zhht.irn.sink.LaneNormSink;
import com.zhht.irn.utils.DruidConnectPoolUtils;
import com.zhht.irn.utils.FlinkUtils;
import org.apache.flink.api.common.eventtime.WatermarkStrategy;
import org.apache.flink.api.common.functions.FilterFunction;
......@@ -26,8 +27,8 @@ public class LaneNormJob {
public static void main(String[] args) throws Exception {
try {
StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
env.setRestartStrategy(RestartStrategies.noRestart());
DataStream<String> cycleStringStream = FlinkUtils.createKafkaStream(args, env,"signal_cycle_data","laneNormJob");
DataStream<String> carStringStream = FlinkUtils.createKafkaStream(args, env,"trips_event_info","laneNormJob");
......@@ -77,9 +78,13 @@ public class LaneNormJob {
DataStream<TravelEvent> carStream = carStringStream.map(getTravelEventFunction).assignTimestampsAndWatermarks(travelEventWatermarkStrategy).keyBy(carKeySelector).filter(filterFunction);
ConnectedStreams<Cycle, TravelEvent> cycleTravelEventConnectedStreams = cycleDataStreamSource.connect(carStream).keyBy(cycle -> cycle.getCrossCode(), travelEvent -> travelEvent.getCrossId());
SingleOutputStreamOperator<LaneNorm> process = cycleTravelEventConnectedStreams.process(new LaneNormProcessFunction());
process.addSink(new LaneNormSink());
SingleOutputStreamOperator<List<LaneNorm>> process = cycleTravelEventConnectedStreams.process(new LaneNormProcessFunction()).setParallelism(4);
process.addSink(new LaneNormSink()).name("laneNormJobSink");
env.execute("laneNormJob");
}catch (Exception e){
System.out.println("程序异常关闭->"+e);
}
}
/*获取信控指标 并计算相关属性*/
......
realtime/hologram-streaming/src/main/java/com/zhht/irn/sink/DirectionEvalSecondSink.java
......@@ -3,6 +3,7 @@ package com.zhht.irn.sink;
import com.zhht.irn.entity.metric.DirectionEvalSecondMetric;
import com.zhht.irn.utils.DateUtils;
import com.zhht.irn.utils.DorisUtils;
import com.zhht.irn.utils.DruidConnectPoolUtils;
import com.zhht.irn.utils.MySQLUtils;
import org.apache.flink.configuration.Configuration;
import org.apache.flink.streaming.api.functions.sink.RichSinkFunction;
......
realtime/hologram-streaming/src/main/java/com/zhht/irn/sink/GreenRadioSink.java
......@@ -4,6 +4,7 @@ import com.zhht.irn.entity.Cycle;
import com.zhht.irn.entity.Stage;
import com.zhht.irn.utils.DateUtils;
import com.zhht.irn.utils.DorisUtils;
import com.zhht.irn.utils.DruidConnectPoolUtils;
import com.zhht.irn.utils.MySQLUtils;
import org.apache.flink.configuration.Configuration;
import org.apache.flink.streaming.api.functions.sink.RichSinkFunction;
......
realtime/hologram-streaming/src/main/java/com/zhht/irn/sink/LaneNormSink.java
......@@ -3,6 +3,7 @@ package com.zhht.irn.sink;
import com.zhht.irn.entity.LaneNorm;
import com.zhht.irn.utils.DateUtils;
import com.zhht.irn.utils.DorisUtils;
import com.zhht.irn.utils.DruidConnectPoolUtils;
import com.zhht.irn.utils.MySQLUtils;
import org.apache.flink.api.java.tuple.Tuple2;
import org.apache.flink.configuration.Configuration;
......@@ -11,6 +12,7 @@ import org.apache.flink.streaming.api.functions.sink.SinkFunction;
import java.sql.Connection;
import java.sql.PreparedStatement;
import java.util.List;
/**
* @Description:
......@@ -19,7 +21,7 @@ import java.sql.PreparedStatement;
* @retrun:
* @Creat :2022-10-31-14:11
**/
public class LaneNormSink extends RichSinkFunction<LaneNorm> {
public class LaneNormSink extends RichSinkFunction<List<LaneNorm>> {
Connection connection;
PreparedStatement insertPstmt;
......@@ -49,7 +51,11 @@ public class LaneNormSink extends RichSinkFunction<LaneNorm> {
* 1000w的数据 1000w次
*/
@Override
public void invoke(LaneNorm value, SinkFunction.Context context) throws Exception {
public void invoke(List<LaneNorm> laneNorms, SinkFunction.Context context) throws Exception {
if(laneNorms.size()==0){
return;
}
for (LaneNorm value: laneNorms){
insertPstmt.setString(1, value.getCorssId());
insertPstmt.setString(2, value.getLaneId());
insertPstmt.setString(3, value.getDirection());
......@@ -69,6 +75,8 @@ public class LaneNormSink extends RichSinkFunction<LaneNorm> {
insertPstmt.setString(17,value.getCycleStartTime());
insertPstmt.setString(18,value.getCycleEndTime());
insertPstmt.setString(19,DateUtils.getTodayTime());
insertPstmt.execute();
insertPstmt.addBatch();
}
insertPstmt.executeBatch();
}
}
realtime/hologram-streaming/src/main/java/com/zhht/irn/sink/TravelMetricSink.java
......@@ -2,6 +2,7 @@ package com.zhht.irn.sink;
import com.zhht.irn.entity.Travel;
import com.zhht.irn.utils.DorisUtils;
import com.zhht.irn.utils.DruidConnectPoolUtils;
import org.apache.flink.configuration.Configuration;
import org.apache.flink.streaming.api.functions.sink.RichSinkFunction;
......
realtime/hologram-streaming/src/main/java/com/zhht/irn/utils/DruidConnectPoolUtils.java
package com.zhht.irn.utils;
import com.alibaba.druid.pool.DruidDataSource;
import com.alibaba.druid.pool.DruidPooledConnection;
import org.apache.flink.api.java.utils.ParameterTool;
import java.io.InputStream;
import java.sql.Connection;
import java.sql.PreparedStatement;
import java.sql.SQLException;
......@@ -12,6 +15,9 @@ import java.sql.SQLException;
* @date 2022-09-05 15:11:00
*/
public class DruidConnectPoolUtils {
private static DruidDataSource pool=null;
/**
* 初始化连接池
* @param dbHosts
......@@ -25,15 +31,66 @@ public class DruidConnectPoolUtils {
dataSource.setUrl(dbHosts);
dataSource.setUsername(username);
dataSource.setPassword(password);
dataSource.setInitialSize(3); //初始连接数，默认0
dataSource.setMaxActive(10); //最大连接数，默认8
dataSource.setMinIdle(5); //最小闲置数
dataSource.setMaxWait(3000);//获取连接的最大等待时间，单位毫秒
dataSource.setTestWhileIdle(true); //指明连接是否被空闲连接回收器(如果有)进行检验.如果检测失败,则连接将被从池中去除
dataSource.setTestOnBorrow(false); // 借出连接时不要测试，否则很影响性能
return dataSource;
}
public static void initDataSource(String dbHosts, String username, String password) {
DruidDataSource dataSource = new DruidDataSource();
dataSource.setDriverClassName("com.mysql.jdbc.Driver");
dataSource.setUrl(dbHosts);
dataSource.setUsername(username);
dataSource.setPassword(password);
dataSource.setInitialSize(3); //初始连接数，默认0
dataSource.setMaxActive(30); //最大连接数，默认8
dataSource.setMaxActive(10); //最大连接数，默认8
dataSource.setMinIdle(5); //最小闲置数
dataSource.setMaxWait(3000);//获取连接的最大等待时间，单位毫秒
dataSource.setTestWhileIdle(true); //指明连接是否被空闲连接回收器(如果有)进行检验.如果检测失败,则连接将被从池中去除
dataSource.setTestOnBorrow(false); // 借出连接时不要测试，否则很影响性能
pool=dataSource;
}
return dataSource;
public static DruidPooledConnection getConnection(){
try {
if(pool==null){
InputStream inputStream = DruidConnectPoolUtils.class.getClassLoader().getResourceAsStream("doris.properties");
ParameterTool tool = ParameterTool.fromPropertiesFile(inputStream);
// 根据路径的配置文件获取MySQL链接参数
String driver = tool.getRequired("driver");
String username = tool.getRequired("username");
String password = tool.getRequired("password");
String host = tool.getRequired("host");
String port = tool.getRequired("port");
String database = tool.get("database", "default");
String jdbcUrl = String.format("jdbc:mysql://%s:%s/%s", host, port, database);
initDataSource(jdbcUrl,username,password);
}
}catch (Exception e){
e.printStackTrace();
}
try {
return pool.getConnection();
} catch (SQLException e) {
e.printStackTrace();
}
return null;
}
public static void colseDataSource() {
try {
if(pool!=null){
pool.close();
}
pool.close();
} catch (Exception e) {
e.printStackTrace();
}
}
/**
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment