迅捷开发环境，spark测试用例提交

需要放到迅捷开发环境上面运行，运行的命令见源码类注释。

ipu-spark-example/pom.xml

@@ -21,6 +21,7 @@
		<ipu>3.1-SNAPSHOT</ipu>
		<jdk>1.8</jdk>
		<junit>4.12</junit>
		<hadoop>2.7.3</hadoop>
		<spark>2.4.1</spark>
		<slf4j-api>1.7.16</slf4j-api>
	</properties>
@@ -42,15 +43,9 @@
		<dependency>
			<groupId>org.apache.hadoop</groupId>
			<artifactId>hadoop-client</artifactId>
			<version>2.7.3</version>
			<version>${hadoop}</version>
		</dependency>
		<!--<dependency>
			<groupId>io.netty</groupId>
			<artifactId>netty-all</artifactId>
			<version>4.1.17.Final</version>
		</dependency>-->
		<dependency>
			<groupId>org.apache.spark</groupId>
			<artifactId>spark-core_2.11</artifactId>
@@ -63,13 +58,6 @@
			<artifactId>slf4j-log4j12</artifactId>
			<version>${slf4j-api}</version>
		</dependency>
		<!-- Failed to set setXIncludeAware(true) for parser -->
		<!--<dependency>
			<groupId>xerces</groupId>
			<artifactId>xercesImpl</artifactId>
			<version>2.9.1</version>
		</dependency>-->
	</dependencies>
	<build>

ipu-spark-example/src/main/java/com/ai/ipu/example/spark/SparkConfig.java

@@ -16,8 +16,6 @@ public class SparkConfig {
    private static String hdfsUri;
    /*测试文件*/
    private static String testFileName;
    /*项目打包路径*/
    private static String jarFilePath;
    public static String getSparkMaster() {
        return sparkMaster;
@@ -31,10 +29,6 @@ public class SparkConfig {
        return testFileName;
    }
    public static String getJarFilePath() {
        return jarFilePath;
    }
    /*加载配置文件*/
    static {
        try {

ipu-spark-example/src/main/java/com/ai/ipu/example/spark/SparkExample.java

@@ -19,11 +19,8 @@ import java.util.List;
 *
 * @author lilb3@asiainfo.com
 * @since 2019-05-24 17:10
 * 部署服务器方式：
 * a) SpringBoot打包部署 需要在pom指定startClass为com.ai.ipu.example.spark.SparkExample
 * $SPARK_HOME/bin/spark-submit --class org.springframework.boot.loader.JarLauncher --master local[4] /home/mysql/test/comp-example-1.0.jar iZm5e5xe1w25avi0io1f5aZ
 * b) 普通打包部署 需要上传原包和依赖包 如框架日志ipu-basic-3.1-SNAPSHOT.jar， 多个jar以逗号隔开
 * $SPARK_HOME/bin/spark-submit --class com.ai.ipu.example.spark.SparkExample --master local[4] --jars /home/mysql/test/ipu-basic-3.1-SNAPSHOT.jar /home/mysql/test/comp-example-1.0.jar.original iZm5e5xe1w25avi0io1f5aZ
 * 迅捷开发环境的shell里面执行：
 * ~/software/spark-2.4.1-bin-hadoop2.7/bin/spark-submit --jars /home/mysql/test/ipu-basic-3.1-SNAPSHOT.jar --class com.ai.ipu.example.spark.SparkExample /home/mysql/test/ipu-spark-example-1.0.jar
 **/
public class SparkExample {
    public static void main(String[] args) {
@@ -35,7 +32,7 @@ public class SparkExample {
         */
        SparkConf sparkConf = new SparkConf().setAppName("SparkExample").setMaster(SparkConfig.getSparkMaster());
        /*java.lang.ClassCastException: cannot assign instance of java.lang.invoke.SerializedLambda to field org.apache.spark.api.java.JavaRDDLike$$anonfun$fn$1$1.f$3 of type org.apache.spark.api.java.function.FlatMapFunction in instance of org.apache.spark.api.java.JavaRDDLike$$anonfun$fn$1$1*/
        sparkConf.setJars(new String[]{SparkConfig.getJarFilePath()});
        //sparkConf.setJars(new String[]{SparkConfig.getJarFilePath()});
        JavaSparkContext jsc = new JavaSparkContext(sparkConf);

ipu-spark-example/src/main/resources/spark.properties

@@ -3,6 +3,4 @@ spark.master=spark://iZm5e5xe1w25avi0io1f5aZ:7077
#Hadoop地址
hdfs.uri=hdfs://iZm5e5xe1w25avi0io1f5aZ:9000
#测试文件
test.file.name=aaa.txt
#项目打包路径
jar.file.path=D:\\ideaws\\rest\\code-example\\ipu-spark-example\\target\\ipu-spark-example-1.0.jar
test.file.name=aaa.txt