Skip to content

Z
ZHHT-IRN-BD-ANALYSIS
Commit 6e52c4b2 by 李凯旋
Options

初始化

parents
Showing with 368 additions and 0 deletions
.gitignore 0 → 100644
.DS_Store
**/.idea/*
**/target/*
.idea
*.iml
*.class
.project
**/.settings/*
**/*/dependency-reduced-pom.xml
holographic-intersection/pom.xml 0 → 100644
<?xml version="1.0" encoding="UTF-8"?>
<project xmlns="http://maven.apache.org/POM/4.0.0"
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
<parent>
<artifactId>ZHHT-IRN-BD-ANALYSIS</artifactId>
<groupId>com.zhht.irn</groupId>
<version>1.0-SNAPSHOT</version>
</parent>
<modelVersion>4.0.0</modelVersion>
<artifactId>holographic-intersection</artifactId>
<properties>
<maven.compiler.source.version>1.8</maven.compiler.source.version>
<maven.compiler.target.version>1.8</maven.compiler.target.version>
<java.version>1.8</java.version>
<spark.version>2.4.0</spark.version>
<scala.version>2.11</scala.version>
<scala.binary.version>2.11.0</scala.binary.version>
</properties>
<dependencies>
<dependency>
<groupId>com.typesafe</groupId>
<artifactId>config</artifactId>
<version>1.2.1</version>
</dependency>
<dependency>
<groupId>org.apache.spark</groupId>
<artifactId>spark-core_${scala.version}</artifactId>
<version>${spark.version}</version>
</dependency>
<dependency>
<groupId>org.apache.spark</groupId>
<artifactId>spark-sql_${scala.version}</artifactId>
<version>${spark.version}</version>
</dependency>
<dependency>
<groupId>org.apache.spark</groupId>
<artifactId>spark-mllib_${scala.version}</artifactId>
<version>${spark.version}</version>
</dependency>
<dependency>
<groupId>org.apache.spark</groupId>
<artifactId>spark-hive_${scala.version}</artifactId>
<version>${spark.version}</version>
</dependency>
<dependency>
<groupId>mysql</groupId>
<artifactId>mysql-connector-java</artifactId>
<version>8.0.27</version>
</dependency>
<dependency>
<groupId>org.scala-lang</groupId>
<artifactId>scala-library</artifactId>
<version>${scala.binary.version}</version>
<scope>compile</scope>
</dependency>
<!-- <dependency>
<groupId>org.apache.commons</groupId>
<artifactId>commons-lang4</artifactId>
<version>4.0</version>
</dependency>-->
<!--<dependency>
<groupId>com.alibaba</groupId>
<artifactId>fastjson</artifactId>
<version>1.2.32</version>
</dependency>
<dependency>
<groupId>org.apache.logging.log4j</groupId>
<artifactId>log4j-api</artifactId>
<version>2.7</version>
</dependency>-->
<!--<dependency>
<groupId>com.fasterxml.woodstox</groupId>
<artifactId>woodstox-core</artifactId>
<version>5.0.3</version>
</dependency>
<dependency>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-common</artifactId>
<version>3.1.0</version>
</dependency>-->
<!-- <dependency>
<groupId>org.apache.flink</groupId>
<artifactId>flink-scala_2.11</artifactId>
<version>1.12.0</version>
<scope>compile</scope>
</dependency>
<dependency>
<groupId>org.apache.flink</groupId>
<artifactId>flink-clients_2.11</artifactId>
<version>1.12.0</version>
</dependency>
&lt;!&ndash; https://mvnrepository.com/artifact/org.apache.flink/flink-streaming-scala &ndash;&gt;
<dependency>
<groupId>org.apache.flink</groupId>
<artifactId>flink-streaming-scala_2.11</artifactId>
<version>1.12.0</version>
<scope>compile</scope>
</dependency>
<dependency>
<groupId>org.apache.flink</groupId>
<artifactId>flink-connector-kafka_2.11</artifactId>
<version>1.12.0</version>
</dependency>
<dependency>
<groupId>org.roaringbitmap</groupId>
<artifactId>RoaringBitmap</artifactId>
<version>0.9.0</version>
</dependency>-->
</dependencies>
<build>
<plugins>
<plugin>
<groupId>org.scala-tools</groupId>
<artifactId>maven-scala-plugin</artifactId>
<version>2.15.2</version>
<executions>
<execution>
<id>scala-compile-first</id>
<goals>
<goal>compile</goal>
</goals>
<configuration>
<includes>
<include>**/*.scala</include>
</includes>
</configuration>
</execution>
<execution>
<id>scala-test-compile</id>
<goals>
<goal>testCompile</goal>
</goals>
</execution>
</executions>
</plugin>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-compiler-plugin</artifactId>
<configuration>
<source>${maven.compiler.source.version}</source>
<target>${maven.compiler.target.version}</target>
</configuration>
</plugin>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-dependency-plugin</artifactId>
<executions>
<execution>
<id>copy</id>
<phase>package</phase>
<goals>
<goal>copy-dependencies</goal>
</goals>
<configuration>
<outputDirectory>${project.build.directory}/lib</outputDirectory>
</configuration>
</execution>
</executions>
</plugin>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-jar-plugin</artifactId>
<version>2.4</version>
<!-- The configuration of the plugin -->
<configuration>
<!-- Configuration of the archiver -->
<archive>
<!--
生成的jar中，不要包含pom.xml和pom.properties这两个文件
-->
<addMavenDescriptor>false</addMavenDescriptor>
<!-- Manifest specific configuration -->
<manifest>
<!--
是否要把第三方jar放到manifest的classpath中
-->
<addClasspath>true</addClasspath>
<!--
生成的manifest中classpath的前缀，因为要把第三方jar放到lib目录下，所以classpath的前缀是lib/
-->
<classpathPrefix>lib/</classpathPrefix>
</manifest>
</archive>
</configuration>
</plugin>
</plugins>
</build>
</project>
\ No newline at end of file
holographic-intersection/src/main/java/TestJava.java 0 → 100644
public class TestJava {
public static void main(String[] args) {
System.out.println("test java");
}
}
holographic-intersection/src/main/resources/application.conf 0 → 100644
##应用程序基础配置
application{
spark:{
##spark应用提交方式，如果未设，将接受spark_submit提交的参数
master:"local[*]"
// master:"yarn"
}
}
commons{
datasource:{
mysql:{
url:"jdbc:mysql://localhost:3307/test?useSSL=false&autoReconnect=true&failOverReadOnly=false&rewriteBatchedStatements=true&useUnicode=true&characterEncoding=utf8"
username:"root"
password:"mima"
driver:"com.mysql.cj.jdbc.Driver"
}
}
}
holographic-intersection/src/main/scala/com/zhht/irn/bigdata/atomic/base_tables.scala 0 → 100644
package com.zhht.irn.bigdata.atomic
import com.zhht.irn.bigdata.config.ApplicationConfig.props
import java.util.{Date, Properties}
import org.apache.spark.SparkConf
import org.apache.spark.sql.functions.{col, max, sum, when}
import org.apache.spark.sql.{DataFrame, SaveMode, SparkSession}
import java.text.SimpleDateFormat
object base_tables {
// 读取信号数据
def get_sign(spark:SparkSession,TableName:String):DataFrame ={
val prop = new Properties()
prop.setProperty("user", props.getString("commons.datasource.mysql.username"))
prop.setProperty("password", props.getString("commons.datasource.mysql.password"))
prop.setProperty("driver", props.getString("commons.datasource.mysql.driver"))
val url = props.getString("commons.datasource.mysql.url")
spark.read.jdbc(url,TableName,prop)
}
// main class
def main(args: Array[String]): Unit = {
val sparkConfig = new SparkConf().setAppName("base_table")
sparkConfig.setMaster("local[*]") //在集群提交运行时，需要修改此行
val spark: SparkSession = SparkSession.builder().config(sparkConfig)
.config("metastore.catalog.default","hive")
.config("hive.strict.managed.tables","false")
.config("hive.create.as.insert.only","false")
.config("metastore.create.as.acid","false")
.config("spark.sql.warehouse.dir", "hdfs://localhost:8020/warehouse/tablespace/managed/hive")
.enableHiveSupport()
.getOrCreate()
val sc = spark.sparkContext
sc.setLogLevel("ERROR")
def NowTime():String = {
val now:Date = new Date()
val fm:SimpleDateFormat = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss")
fm.format(now)
}
// project begin
println("Start Time:"+NowTime())
get_sign(spark,"dic_phase").show()
println("Finished Time:"+NowTime())
spark.stop()
sc.stop()
System.exit(0)
}
}
holographic-intersection/src/main/scala/com/zhht/irn/bigdata/config/ApplicationConfig.scala 0 → 100644
package com.zhht.irn.bigdata.config
import com.typesafe.config.ConfigFactory
object ApplicationConfig {
val props = ConfigFactory.load("application.conf") //读取配置文件，获取配置信息
val master = props.getString("application.spark.master")
val username_mysql = props.getString("commons.datasource.mysql.username")
val password_mysql = props.getString("commons.datasource.mysql.password")
val url_mysql = props.getString("commons.datasource.mysql.url")
val driver_mysql = props.getString("commons.datasource.mysql.driver")
/**
* 加载关系数据库(Mysql)配置
* @param tableName
* @return
*/
def getMysqlOptions(tableName:String):Map[String, String] = {
val jdbcMapv = Map("url" -> url_mysql,
"user" -> username_mysql,
"password" -> password_mysql,
"dbtable" -> tableName,
"driver" -> driver_mysql)
return jdbcMapv
}
}
pom.xml 0 → 100644
<?xml version="1.0" encoding="UTF-8"?>
<project xmlns="http://maven.apache.org/POM/4.0.0"
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
<modelVersion>4.0.0</modelVersion>
<groupId>com.zhht.irn</groupId>
<artifactId>ZHHT-IRN-BD-ANALYSIS</artifactId>
<packaging>pom</packaging>
<version>1.0-SNAPSHOT</version>
<modules>
<module>holographic-intersection</module>
</modules>
<properties>
<maven.compiler.source>8</maven.compiler.source>
<maven.compiler.target>8</maven.compiler.target>
</properties>
</project>
\ No newline at end of file
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment