From 458f57759a201017bd963957dea7a0c25790e050 Mon Sep 17 00:00:00 2001
From: luoxiang <2806718453@qq.com>
Date: Wed, 15 May 2019 07:37:02 +0800
Subject: [PATCH] modify

---
 code/spark/spark-core/pom.xml                   |  6 ++++++
 .../main/java/rdd/java/TransformationTest.java  | 12 ++++++++++--
 .../src/main/java/rdd/scala/Test.scala          | 14 --------------
 .../java/rdd/scala/TransformationTest.scala     | 17 +++++++++--------
 4 files changed, 25 insertions(+), 24 deletions(-)
 delete mode 100644 code/spark/spark-core/src/main/java/rdd/scala/Test.scala

diff --git a/code/spark/spark-core/pom.xml b/code/spark/spark-core/pom.xml
index d805edd..139270c 100644
--- a/code/spark/spark-core/pom.xml
+++ b/code/spark/spark-core/pom.xml
@@ -43,6 +43,12 @@
             <artifactId>junit</artifactId>
             <version>4.12</version>
         </dependency>
+        <!--Exception in thread "main" java.lang.ArrayIndexOutOfBoundsException: 10582-->
+        <dependency>
+            <groupId>com.thoughtworks.paranamer</groupId>
+            <artifactId>paranamer</artifactId>
+            <version>2.8</version>
+        </dependency>
     </dependencies>
 
 
diff --git a/code/spark/spark-core/src/main/java/rdd/java/TransformationTest.java b/code/spark/spark-core/src/main/java/rdd/java/TransformationTest.java
index 80a2671..19a21bf 100644
--- a/code/spark/spark-core/src/main/java/rdd/java/TransformationTest.java
+++ b/code/spark/spark-core/src/main/java/rdd/java/TransformationTest.java
@@ -12,19 +12,27 @@ import java.util.List;
 
 public class TransformationTest {
 
+
     private static JavaSparkContext sc = null;
 
 
     @Before
     public void prepare() {
-        SparkConf conf = new SparkConf().setMaster("local[2]").setAppName("TransformationTest");
+        SparkConf conf = new SparkConf().setAppName("TransformationTest").setMaster("local[2]");
         sc = new JavaSparkContext(conf);
     }
 
     @Test
     public void map() {
         List<Integer> list = Arrays.asList(3, 6, 9, 10, 12, 21);
-        sc.parallelize(list).map(x -> x * 10).foreach(System.out::println);
+        /*
+         * 不要使用方法引用的形式 : System.out::println , 否则会抛出下面的异常:
+         * org.apache.spark.SparkException: Task not serializable
+         * Caused by: java.io.NotSerializableException: java.io.PrintStream
+         * 这是由于Spark程序中map、foreach等算子内部引用了类成员函数或变量时,需要该类所有成员都支持序列化,
+         * 如果该类某些成员变量不支持序列化，就会抛出上面的异常
+         */
+        sc.parallelize(list).map(x -> x * 10).foreach(x -> System.out.println(x));
     }
 
 
diff --git a/code/spark/spark-core/src/main/java/rdd/scala/Test.scala b/code/spark/spark-core/src/main/java/rdd/scala/Test.scala
deleted file mode 100644
index a8e3af9..0000000
--- a/code/spark/spark-core/src/main/java/rdd/scala/Test.scala
+++ /dev/null
@@ -1,14 +0,0 @@
-package rdd.scala
-
-import org.apache.spark.{SparkConf, SparkContext}
-
-object Test extends App {
-
-
-  val conf = new SparkConf().setAppName("TransformationTest123").setMaster("local[2]")
-  val sc = new SparkContext(conf)
-
-  val list = List(3, 6, 9, 10, 12, 21)
-  sc.parallelize(list).map(_ * 10).foreach(println)
-
-}
diff --git a/code/spark/spark-core/src/main/java/rdd/scala/TransformationTest.scala b/code/spark/spark-core/src/main/java/rdd/scala/TransformationTest.scala
index 5ed96aa..8fcd2a1 100644
--- a/code/spark/spark-core/src/main/java/rdd/scala/TransformationTest.scala
+++ b/code/spark/spark-core/src/main/java/rdd/scala/TransformationTest.scala
@@ -1,17 +1,13 @@
 package rdd.scala
 
 import org.apache.spark.{SparkConf, SparkContext}
-import org.junit.{Before, Test}
+import org.junit.{After, Test}
 
-class TransformationTest extends {
+class TransformationTest {
 
-  var sc: SparkContext = _
+  val conf: SparkConf = new SparkConf().setAppName("TransformationTest").setMaster("local[2]")
+  val sc = new SparkContext(conf)
 
-  @Before
-  def prepare(): Unit = {
-    val conf = new SparkConf().setAppName("TransformationTest").setMaster("local[2]")
-    sc = new SparkContext(conf)
-  }
 
   @Test
   def map(): Unit = {
@@ -19,5 +15,10 @@ class TransformationTest extends {
     sc.parallelize(list).map(_ * 10).foreach(println)
   }
 
+  @After
+  def destroy(): Unit = {
+    sc.stop()
+  }
+
 
 }
\ No newline at end of file