Merge remote-tracking branch 'origin/master'

realtime/hologram-streaming/src/main/java/com/zhht/irn/functions/TravelEventAndCycleCoProcessFunction.java

@@ -87,16 +87,16 @@ public class TravelEventAndCycleCoProcessFunction extends CoProcessFunction<Trav
         carRecordState = getRuntimeContext().getMapState(carRecordStateDescriptor);
 
 
-        dataSource = DruidConnectPoolUtils.getDataSource
+        connection = DruidConnectPoolUtils.getConnection();
             //    ("jdbc:mysql://localhost:9030/demo?characterEncoding=utf8",
             //    ("jdbc:mysql://localhost:3307/test?characterEncoding=utf8",
             //    ("jdbc:mysql://10.243.0.26:3306/test?characterEncoding=utf8",
-                ("jdbc:mysql://10.243.0.22:9030/demo?characterEncoding=utf8",
+               /* ("jdbc:mysql://10.243.0.22:9030/demo?characterEncoding=utf8",
                         "root",
                 //        "root123");
                         "mima");
-
-        connection = dataSource.getConnection();
+*/
+       // dataSource.getConnection();
         dim_cnt_cross_lane_position = getLaneDimData(connection);
 
     }
@@ -754,7 +754,7 @@ public class TravelEventAndCycleCoProcessFunction extends CoProcessFunction<Trav
      */
     private void sinkTravelLineSinkInfoToMysql(List<TravelLineSinkInfo> list) throws SQLException {
 
-        DruidPooledConnection connection = dataSource.getConnection();
+        DruidPooledConnection connection = DruidConnectPoolUtils.getConnection();
 
         String sql = "INSERT INTO demo.app_cross_line_travel_car_info" +
                 "(record_date,cross_id,statistic_time,cycle_begin_time,cycle_end_time, lane_id, cycle_order, direction," +

realtime/hologram-streaming/src/main/java/com/zhht/irn/job/LaneNormJob.java

@@ -5,6 +5,7 @@ import com.zhht.irn.entity.*;
 import com.zhht.irn.enums.EventType;
 import com.zhht.irn.functions.LaneNormProcessFunction;
 import com.zhht.irn.sink.LaneNormSink;
+import com.zhht.irn.utils.DruidConnectPoolUtils;
 import com.zhht.irn.utils.FlinkUtils;
 import org.apache.flink.api.common.eventtime.WatermarkStrategy;
 import org.apache.flink.api.common.functions.FilterFunction;
@@ -26,8 +27,8 @@ public class LaneNormJob {
 
 
     public static void main(String[] args) throws Exception {
+        try {
             StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
-
             env.setRestartStrategy(RestartStrategies.noRestart());
             DataStream<String> cycleStringStream = FlinkUtils.createKafkaStream(args, env,"signal_cycle_data","laneNormJob");
             DataStream<String> carStringStream = FlinkUtils.createKafkaStream(args, env,"trips_event_info","laneNormJob");
@@ -77,9 +78,13 @@ public class LaneNormJob {
             DataStream<TravelEvent> carStream = carStringStream.map(getTravelEventFunction).assignTimestampsAndWatermarks(travelEventWatermarkStrategy).keyBy(carKeySelector).filter(filterFunction);
 
             ConnectedStreams<Cycle, TravelEvent> cycleTravelEventConnectedStreams = cycleDataStreamSource.connect(carStream).keyBy(cycle -> cycle.getCrossCode(), travelEvent -> travelEvent.getCrossId());
-        SingleOutputStreamOperator<LaneNorm> process = cycleTravelEventConnectedStreams.process(new LaneNormProcessFunction());
-        process.addSink(new LaneNormSink());
+            SingleOutputStreamOperator<List<LaneNorm>> process = cycleTravelEventConnectedStreams.process(new LaneNormProcessFunction()).setParallelism(4);
+            process.addSink(new LaneNormSink()).name("laneNormJobSink");
             env.execute("laneNormJob");
+        }catch (Exception e){
+            System.out.println("程序异常关闭->"+e);
+        }
+
     }
 
     /*获取信控指标 并计算相关属性*/

realtime/hologram-streaming/src/main/java/com/zhht/irn/sink/DirectionEvalSecondSink.java

@@ -3,6 +3,7 @@ package com.zhht.irn.sink;
 import com.zhht.irn.entity.metric.DirectionEvalSecondMetric;
 import com.zhht.irn.utils.DateUtils;
 import com.zhht.irn.utils.DorisUtils;
+import com.zhht.irn.utils.DruidConnectPoolUtils;
 import com.zhht.irn.utils.MySQLUtils;
 import org.apache.flink.configuration.Configuration;
 import org.apache.flink.streaming.api.functions.sink.RichSinkFunction;

realtime/hologram-streaming/src/main/java/com/zhht/irn/sink/GreenRadioSink.java

@@ -4,6 +4,7 @@ import com.zhht.irn.entity.Cycle;
 import com.zhht.irn.entity.Stage;
 import com.zhht.irn.utils.DateUtils;
 import com.zhht.irn.utils.DorisUtils;
+import com.zhht.irn.utils.DruidConnectPoolUtils;
 import com.zhht.irn.utils.MySQLUtils;
 import org.apache.flink.configuration.Configuration;
 import org.apache.flink.streaming.api.functions.sink.RichSinkFunction;

realtime/hologram-streaming/src/main/java/com/zhht/irn/sink/LaneNormSink.java

@@ -3,6 +3,7 @@ package com.zhht.irn.sink;
 import com.zhht.irn.entity.LaneNorm;
 import com.zhht.irn.utils.DateUtils;
 import com.zhht.irn.utils.DorisUtils;
+import com.zhht.irn.utils.DruidConnectPoolUtils;
 import com.zhht.irn.utils.MySQLUtils;
 import org.apache.flink.api.java.tuple.Tuple2;
 import org.apache.flink.configuration.Configuration;
@@ -11,6 +12,7 @@ import org.apache.flink.streaming.api.functions.sink.SinkFunction;
 
 import java.sql.Connection;
 import java.sql.PreparedStatement;
+import java.util.List;
 
 /**
  * @Description:
@@ -19,7 +21,7 @@ import java.sql.PreparedStatement;
  * @retrun:
  * @Creat :2022-10-31-14:11
  **/
-public class LaneNormSink  extends RichSinkFunction<LaneNorm> {
+public class LaneNormSink  extends RichSinkFunction<List<LaneNorm>> {
 
     Connection connection;
     PreparedStatement insertPstmt;
@@ -49,7 +51,11 @@ public class LaneNormSink  extends RichSinkFunction<LaneNorm> {
      * 1000w的数据  1000w次
      */
     @Override
-    public void invoke(LaneNorm value, SinkFunction.Context context) throws Exception {
+    public void invoke(List<LaneNorm> laneNorms, SinkFunction.Context context) throws Exception {
+        if(laneNorms.size()==0){
+            return;
+        }
+        for (LaneNorm value: laneNorms){
             insertPstmt.setString(1, value.getCorssId());
             insertPstmt.setString(2, value.getLaneId());
             insertPstmt.setString(3, value.getDirection());
@@ -69,6 +75,8 @@ public class LaneNormSink  extends RichSinkFunction<LaneNorm> {
             insertPstmt.setString(17,value.getCycleStartTime());
             insertPstmt.setString(18,value.getCycleEndTime());
             insertPstmt.setString(19,DateUtils.getTodayTime());
-        insertPstmt.execute();
+            insertPstmt.addBatch();
+        }
+        insertPstmt.executeBatch();
     }
 }

realtime/hologram-streaming/src/main/java/com/zhht/irn/sink/TravelMetricSink.java

@@ -2,6 +2,7 @@ package com.zhht.irn.sink;
 
 import com.zhht.irn.entity.Travel;
 import com.zhht.irn.utils.DorisUtils;
+import com.zhht.irn.utils.DruidConnectPoolUtils;
 import org.apache.flink.configuration.Configuration;
 import org.apache.flink.streaming.api.functions.sink.RichSinkFunction;
 

realtime/hologram-streaming/src/main/java/com/zhht/irn/utils/DruidConnectPoolUtils.java

 package com.zhht.irn.utils;
 
 import com.alibaba.druid.pool.DruidDataSource;
+import com.alibaba.druid.pool.DruidPooledConnection;
+import org.apache.flink.api.java.utils.ParameterTool;
 
+import java.io.InputStream;
 import java.sql.Connection;
 import java.sql.PreparedStatement;
 import java.sql.SQLException;
@@ -12,6 +15,9 @@ import java.sql.SQLException;
  * @date 2022-09-05 15:11:00
  */
 public class DruidConnectPoolUtils {
+
+    private static DruidDataSource pool=null;
+
     /**
      * 初始化连接池
      * @param dbHosts
@@ -25,15 +31,66 @@ public class DruidConnectPoolUtils {
         dataSource.setUrl(dbHosts);
         dataSource.setUsername(username);
         dataSource.setPassword(password);
+        dataSource.setInitialSize(3);  //初始连接数，默认0
+        dataSource.setMaxActive(10);  //最大连接数，默认8
+        dataSource.setMinIdle(5);  //最小闲置数
+        dataSource.setMaxWait(3000);//获取连接的最大等待时间，单位毫秒
+        dataSource.setTestWhileIdle(true); //指明连接是否被空闲连接回收器(如果有)进行检验.如果检测失败,则连接将被从池中去除
+        dataSource.setTestOnBorrow(false); // 借出连接时不要测试，否则很影响性能
+        return dataSource;
+    }
+
 
+    public static void initDataSource(String dbHosts, String username, String password) {
+        DruidDataSource dataSource = new DruidDataSource();
+        dataSource.setDriverClassName("com.mysql.jdbc.Driver");
+        dataSource.setUrl(dbHosts);
+        dataSource.setUsername(username);
+        dataSource.setPassword(password);
         dataSource.setInitialSize(3);  //初始连接数，默认0
-        dataSource.setMaxActive(30);  //最大连接数，默认8
+        dataSource.setMaxActive(10);  //最大连接数，默认8
         dataSource.setMinIdle(5);  //最小闲置数
         dataSource.setMaxWait(3000);//获取连接的最大等待时间，单位毫秒
         dataSource.setTestWhileIdle(true); //指明连接是否被空闲连接回收器(如果有)进行检验.如果检测失败,则连接将被从池中去除
         dataSource.setTestOnBorrow(false); // 借出连接时不要测试，否则很影响性能
+        pool=dataSource;
+    }
 
-        return dataSource;
+    public static DruidPooledConnection getConnection(){
+        try {
+            if(pool==null){
+                InputStream inputStream = DruidConnectPoolUtils.class.getClassLoader().getResourceAsStream("doris.properties");
+                ParameterTool tool = ParameterTool.fromPropertiesFile(inputStream);
+                // 根据路径的配置文件获取MySQL链接参数
+                String driver = tool.getRequired("driver");
+                String username = tool.getRequired("username");
+                String password = tool.getRequired("password");
+                String host = tool.getRequired("host");
+                String port = tool.getRequired("port");
+                String database = tool.get("database", "default");
+                String jdbcUrl = String.format("jdbc:mysql://%s:%s/%s", host, port, database);
+                initDataSource(jdbcUrl,username,password);
+            }
+        }catch (Exception e){
+            e.printStackTrace();
+        }
+        try {
+            return pool.getConnection();
+        } catch (SQLException e) {
+            e.printStackTrace();
+        }
+        return null;
+    }
+
+    public static void colseDataSource() {
+        try {
+            if(pool!=null){
+                pool.close();
+            }
+           pool.close();
+        } catch (Exception e) {
+            e.printStackTrace();
+        }
     }
 
     /**