diff --git a/code/spark/spark-core/README.md b/code/spark/spark-core/README.md new file mode 100644 index 0000000..e1cd8da --- /dev/null +++ b/code/spark/spark-core/README.md @@ -0,0 +1,27 @@ +val list = List(3,6,9,10,12,21) +val listRDD = sc.parallelize(list) +val intsRDD = listRDD.map(_*10) +intsRDD.foreach(println) + +sc.parallelize(list).map(_*10).foreach(println) + + +sc.parallelize(list).filter(_>=10).foreach(println) + +val list = List(List(1, 2), List(3), List(), List(4, 5)) +sc.parallelize(list).flatMap(_.toList).map(_*10).foreach(println) + + +val list = List(1,2,3,4,5) +sc.parallelize(list).reduce((x,y) => x+y) +sc.parallelize(list).reduce(_+_) + + +val list = List(("hadoop", 2), ("spark", 3), ("spark", 5), ("storm", 6),("hadoop", 2)) +sc.parallelize(list).reduceByKey(_+_).foreach(println) + + + + val list = List(("hadoop", 2), ("spark", 3), ("spark", 5), ("storm", 6),("hadoop", 2)) +sc.parallelize(list).groupByKey().map(x=>(x._1,x._2.toList)).foreach(println) + diff --git a/code/spark/spark-core/pom.xml b/code/spark/spark-core/pom.xml new file mode 100644 index 0000000..d805edd --- /dev/null +++ b/code/spark/spark-core/pom.xml @@ -0,0 +1,49 @@ + + + 4.0.0 + + com.heibaiying + spark-core + 1.0 + + + 2.12.8 + + + + + + org.apache.maven.plugins + maven-compiler-plugin + + 8 + 8 + + + + + + + + + org.apache.spark + spark-core_2.12 + 2.4.0 + + + org.scalatest + scalatest_2.12 + 3.0.1 + test + + + junit + junit + 4.12 + + + + + \ No newline at end of file diff --git a/code/spark/spark-core/src/main/java/rdd/java/TransformationTest.java b/code/spark/spark-core/src/main/java/rdd/java/TransformationTest.java new file mode 100644 index 0000000..80a2671 --- /dev/null +++ b/code/spark/spark-core/src/main/java/rdd/java/TransformationTest.java @@ -0,0 +1,36 @@ +package rdd.java; + +import org.apache.spark.SparkConf; +import org.apache.spark.api.java.JavaSparkContext; +import org.junit.After; +import org.junit.Before; +import org.junit.Test; + +import java.util.Arrays; +import java.util.List; + + +public class TransformationTest { + + private static JavaSparkContext sc = null; + + + @Before + public void prepare() { + SparkConf conf = new SparkConf().setMaster("local[2]").setAppName("TransformationTest"); + sc = new JavaSparkContext(conf); + } + + @Test + public void map() { + List list = Arrays.asList(3, 6, 9, 10, 12, 21); + sc.parallelize(list).map(x -> x * 10).foreach(System.out::println); + } + + + @After + public void destroy() { + sc.close(); + } + +} diff --git a/code/spark/spark-core/src/main/java/rdd/scala/Test.scala b/code/spark/spark-core/src/main/java/rdd/scala/Test.scala new file mode 100644 index 0000000..a8e3af9 --- /dev/null +++ b/code/spark/spark-core/src/main/java/rdd/scala/Test.scala @@ -0,0 +1,14 @@ +package rdd.scala + +import org.apache.spark.{SparkConf, SparkContext} + +object Test extends App { + + + val conf = new SparkConf().setAppName("TransformationTest123").setMaster("local[2]") + val sc = new SparkContext(conf) + + val list = List(3, 6, 9, 10, 12, 21) + sc.parallelize(list).map(_ * 10).foreach(println) + +} diff --git a/code/spark/spark-core/src/main/java/rdd/scala/TransformationTest.scala b/code/spark/spark-core/src/main/java/rdd/scala/TransformationTest.scala new file mode 100644 index 0000000..5ed96aa --- /dev/null +++ b/code/spark/spark-core/src/main/java/rdd/scala/TransformationTest.scala @@ -0,0 +1,23 @@ +package rdd.scala + +import org.apache.spark.{SparkConf, SparkContext} +import org.junit.{Before, Test} + +class TransformationTest extends { + + var sc: SparkContext = _ + + @Before + def prepare(): Unit = { + val conf = new SparkConf().setAppName("TransformationTest").setMaster("local[2]") + sc = new SparkContext(conf) + } + + @Test + def map(): Unit = { + val list = List(3, 6, 9, 10, 12, 21) + sc.parallelize(list).map(_ * 10).foreach(println) + } + + +} \ No newline at end of file diff --git a/notes/Spark-Transformationå’ŒAction.md b/notes/Spark-Transformationå’ŒAction.md new file mode 100644 index 0000000..e69de29