Merge remote-tracking branch 'origin/master'

realtime/hologram-streaming/src/main/java/com/zhht/irn/consts/Constant.java

@@ -16,5 +16,5 @@ public class Constant {
     // 周期计算的延迟时间 15秒
     public static final int CYCLE_CALCULATE_LATENESS = 15 * 1000;
 
-    public static final int INSERT_BATCH_SIZE = 1000;
+    public static final int INSERT_BATCH_SIZE = 100;
 }

realtime/hologram-streaming/src/main/java/com/zhht/irn/functions/CalLaneMetricProcessFunction2.java

-package com.zhht.irn.functions;
-
-import com.zhht.irn.entity.Cycle;
-import com.zhht.irn.entity.TravelEvent;
-import org.apache.flink.api.common.state.MapState;
-import org.apache.flink.api.common.state.MapStateDescriptor;
-import org.apache.flink.configuration.Configuration;
-import org.apache.flink.streaming.api.functions.KeyedProcessFunction;
-import org.apache.flink.streaming.api.functions.co.KeyedCoProcessFunction;
-import org.apache.flink.util.Collector;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.util.ArrayList;
-import java.util.List;
-
-/**
- * 计算车道级指标的函数
- *
- */
-public class CalLaneMetricProcessFunction2 extends KeyedCoProcessFunction<String, Cycle, TravelEvent, Object> {
-    private static final Logger logger = LoggerFactory.getLogger(CalLaneMetricProcessFunction2.class);
-    // state 管理当前周期中存储的旅行事件
-    private MapState<String, List<TravelEvent>> cycleTravel;
-
-    @Override
-    public void open(Configuration parameters) throws Exception {
-        super.open(parameters);
-        cycleTravel = getRuntimeContext().getMapState(new MapStateDescriptor("cycleTravel", String.class, Object.class));
-    }
-
-    @Override
-    public void processElement1(Cycle cycle, KeyedCoProcessFunction<String, Cycle, TravelEvent, Object>.Context ctx, Collector<Object> out) throws Exception {
-        logger.info("进来个周期数据了：" + cycle.toString());
-        // 在此进行指标计算
-    }
-
-    @Override
-    public void processElement2(TravelEvent event, KeyedCoProcessFunction<String, Cycle, TravelEvent, Object>.Context ctx, Collector<Object> out) throws Exception {
-        logger.info("进来个事件数据了：" + event.toString());
-        if(cycleTravel.get(event.getCrossId()) == null) {
-            cycleTravel.put(event.getCrossId(), new ArrayList<>());
-        }
-        cycleTravel.get(event.getCrossId()).add(event);
-
-    }
-}

realtime/hologram-streaming/src/main/java/com/zhht/irn/job/DirectionEvalJob.java

@@ -4,15 +4,20 @@ import com.alibaba.fastjson.JSON;
 import com.zhht.irn.consts.Constant;
 import com.zhht.irn.entity.Cycle;
 import com.zhht.irn.entity.TravelEvent;
+import com.zhht.irn.entity.metric.DirectionEvalSecondMetric;
 import com.zhht.irn.functions.CalDirectionSecondMetricFunction;
 import com.zhht.irn.sink.DirectionEvalSecondSink;
+import com.zhht.irn.source.CycleMockSource;
+import com.zhht.irn.source.TravelEventMockSource;
 import com.zhht.irn.utils.FlinkUtils;
 import com.zhht.irn.utils.StringUtils;
 import org.apache.flink.api.common.eventtime.WatermarkStrategy;
 import org.apache.flink.streaming.api.datastream.DataStream;
 import org.apache.flink.streaming.api.datastream.KeyedStream;
 import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment;
+import org.apache.flink.streaming.api.functions.co.KeyedCoProcessFunction;
 import org.apache.flink.streaming.api.windowing.time.Time;
+import org.apache.flink.util.Collector;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
@@ -38,7 +43,7 @@ public class DirectionEvalJob {
         DataStream<Cycle> cycleStream =
                 FlinkUtils.createKafkaStream(args, env, Constant.CYCLE_TOPIC_NAME, "DirectionEvalJob")
                 .map(record -> {
-                    System.out.println("信控周期->" + record);
+                    System.out.println(record);
                     return JSON.parseObject(record, Cycle.class);
                 });
 
@@ -46,7 +51,7 @@ public class DirectionEvalJob {
         DataStream<TravelEvent> travelEventStream =
                 FlinkUtils.createKafkaStream(args, env, Constant.TRAVEL_EVENT_TOPIC_NAME, "DirectionEvalJob")
                 .map(record -> {
-                    System.out.println("旅行事件->" + record);
+                    System.out.println(record);
                     return JSON.parseObject(record, TravelEvent.class);
                 }).filter(event -> {
                     return StringUtils.isNotEmpty(event.getEventLineId());
@@ -69,10 +74,9 @@ public class DirectionEvalJob {
         // 周期数据join 旅行事件
         cycleStream.assignTimestampsAndWatermarks(cycleWatermarkStrategy)
                 .keyBy(record -> record.getCrossCode())
-                .intervalJoin(travelEventKeyedStream)
-                .between(Time.minutes(-9), Time.minutes(1))
+                .connect(travelEventKeyedStream)
                 .process(new CalDirectionSecondMetricFunction())
-                .addSink(new DirectionEvalSecondSink());
+                .addSink(new DirectionEvalSecondSink()).setParallelism(4);
 
         // 旅行数据流
         env.execute("DirectionEvalJob");

realtime/hologram-streaming/src/main/java/com/zhht/irn/sink/DirectionEvalSecondSink.java

@@ -44,7 +44,6 @@ public class DirectionEvalSecondSink extends RichSinkFunction<DirectionEvalSecon
 
     @Override
     public void invoke(DirectionEvalSecondMetric metric, Context context) throws Exception {
-        System.out.println("===== insert metric {" + metric + "}");
         insertPstmt.setString(1, metric.getCrossId());
         insertPstmt.setString(2, metric.getLineId());
         insertPstmt.setLong(3, metric.getCycleOrder());

realtime/hologram-streaming/src/main/java/com/zhht/irn/sink/TravelMetricSink.java

@@ -38,6 +38,7 @@ public class TravelMetricSink extends RichSinkFunction<Travel> {
     }
     @Override
     public void close() throws Exception {
+        System.out.println("执行close方法了");
         super.close();
         if(insertPstmt != null) insertPstmt.close();
         if(connection != null) connection.close();
@@ -45,7 +46,6 @@ public class TravelMetricSink extends RichSinkFunction<Travel> {
 
     @Override
     public void invoke(Travel travel, Context context) throws Exception {
-        System.out.println("=====insert travel ======" + travel);
         insertPstmt.setString(1, travel.getRecordId());
         insertPstmt.setString(2, travel.getCrossId());
         insertPstmt.setDate(3, new Date(Long.parseLong(travel.getRecordTime())));

realtime/hologram-streaming/src/main/resources/kafka.properties

 bootstrap.servers=139.9.157.176:9092
 kafka.input.topics=trips_info
 auto.offset.reset=latest
-group.id=wuyanfei
+group.id=TravelInfoJob
 checkpoint.path=file:///data/flink/state
-checkpoint.interval=5000
+checkpoint.interval=60000
 enable.auto.commit=false
\ No newline at end of file

realtime/hologram-streaming/src/main/resources/mysql.properties

 driver=com.mysql.jdbc.Driver
-host=127.0.0.1
-port=13305
+host=10.243.0.26
+port=3306
 username=root
 password=mima
 database=test