modify

code/spark/spark-core/file/emp.json

@@ -0,0 +1,14 @@
+{"EMPNO": 7369,"ENAME": "SMITH","JOB": "CLERK","MGR": 7902,"HIREDATE": "1980-12-17 00:00:00","SAL": 800.00,"COMM": null,"DEPTNO": 20}
+{"EMPNO": 7499,"ENAME": "ALLEN","JOB": "SALESMAN","MGR": 7698,"HIREDATE": "1981-02-20 00:00:00","SAL": 1600.00,"COMM": 300.00,"DEPTNO": 30}
+{"EMPNO": 7521,"ENAME": "WARD","JOB": "SALESMAN","MGR": 7698,"HIREDATE": "1981-02-22 00:00:00","SAL": 1250.00,"COMM": 500.00,"DEPTNO": 30}
+{"EMPNO": 7566,"ENAME": "JONES","JOB": "MANAGER","MGR": 7839,"HIREDATE": "1981-04-02 00:00:00","SAL": 2975.00,"COMM": null,"DEPTNO": 20}
+{"EMPNO": 7654,"ENAME": "MARTIN","JOB": "SALESMAN","MGR": 7698,"HIREDATE": "1981-09-28 00:00:00","SAL": 1250.00,"COMM": 1400.00,"DEPTNO": 30}
+{"EMPNO": 7698,"ENAME": "BLAKE","JOB": "MANAGER","MGR": 7839,"HIREDATE": "1981-05-01 00:00:00","SAL": 2850.00,"COMM": null,"DEPTNO": 30}
+{"EMPNO": 7782,"ENAME": "CLARK","JOB": "MANAGER","MGR": 7839,"HIREDATE": "1981-06-09 00:00:00","SAL": 2450.00,"COMM": null,"DEPTNO": 10}
+{"EMPNO": 7788,"ENAME": "SCOTT","JOB": "ANALYST","MGR": 7566,"HIREDATE": "1987-04-19 00:00:00","SAL": 1500.00,"COMM": null,"DEPTNO": 20}
+{"EMPNO": 7839,"ENAME": "KING","JOB": "PRESIDENT","MGR": null,"HIREDATE": "1981-11-17 00:00:00","SAL": 5000.00,"COMM": null,"DEPTNO": 10}
+{"EMPNO": 7844,"ENAME": "TURNER","JOB": "SALESMAN","MGR": 7698,"HIREDATE": "1981-09-08 00:00:00","SAL": 1500.00,"COMM": 0.00,"DEPTNO": 30}
+{"EMPNO": 7876,"ENAME": "ADAMS","JOB": "CLERK","MGR": 7788,"HIREDATE": "1987-05-23 00:00:00","SAL": 1100.00,"COMM": null,"DEPTNO": 20}
+{"EMPNO": 7900,"ENAME": "JAMES","JOB": "CLERK","MGR": 7698,"HIREDATE": "1981-12-03 00:00:00","SAL": 950.00,"COMM": null,"DEPTNO": 30}
+{"EMPNO": 7902,"ENAME": "FORD","JOB": "ANALYST","MGR": 7566,"HIREDATE": "1981-12-03 00:00:00","SAL": 3000.00,"COMM": null,"DEPTNO": 20}
+{"EMPNO": 7934,"ENAME": "MILLER","JOB": "CLERK","MGR": 7782,"HIREDATE": "1982-01-23 00:00:00","SAL": 1300.00,"COMM": null,"DEPTNO": 10}

code/spark/spark-core/src/main/java/rdd/scala/SparkSqlApp.scala

@@ -0,0 +1,65 @@
+package rdd.scala
+
+import org.apache.spark.sql.expressions.Aggregator
+import org.apache.spark.sql.{Encoder, Encoders, SparkSession, functions}
+
+// 1.定义员工类,对于可能存在null值的字段需要使用Option进行包装
+case class Emp(ename: String, comm: scala.Option[Double], deptno: Long, empno: Long,
+               hiredate: String, job: String, mgr: scala.Option[Long], sal: Double)
+
+// 2.定义聚合操作的中间输出类型
+case class SumAndCount(var sum: Double, var count: Long)
+
+/* 3.自定义聚合函数
+ * @IN  聚合操作的输入类型
+ * @BUF reduction操作输出值的类型
+ * @OUT 聚合操作的输出类型
+ */
+object MyAverage extends Aggregator[Emp, SumAndCount, Double] {
+
+
+  // 4.用于聚合操作的的初始零值
+  override def zero: SumAndCount = SumAndCount(0, 0)
+
+
+  // 5.同一分区中的reduce操作
+  override def reduce(avg: SumAndCount, emp: Emp): SumAndCount = {
+    avg.sum += emp.sal
+    avg.count += 1
+    avg
+  }
+
+  // 6.不同分区中的merge操作
+  override def merge(avg1: SumAndCount, avg2: SumAndCount): SumAndCount = {
+    avg1.sum += avg2.sum
+    avg1.count += avg2.count
+    avg1
+  }
+
+  // 7.定义最终的输出类型
+  override def finish(reduction: SumAndCount): Double = reduction.sum / reduction.count
+
+  // 8.中间类型的编码转换
+  override def bufferEncoder: Encoder[SumAndCount] = Encoders.product
+
+  // 9.输出类型的编码转换
+  override def outputEncoder: Encoder[Double] = Encoders.scalaDouble
+}
+
+object SparkSqlApp {
+
+  // 测试方法
+  def main(args: Array[String]): Unit = {
+
+    val spark = SparkSession.builder().appName("Spark-SQL").master("local[2]").getOrCreate()
+    import spark.implicits._
+    val ds = spark.read.json("file/emp.json").as[Emp]
+
+    // 10.使用内置avg()函数和自定义函数分别进行计算，验证自定义函数是否正确
+    val myAvg = ds.select(MyAverage.toColumn.name("average_sal")).first()
+    val avg = ds.select(functions.avg(ds.col("sal"))).first().get(0)
+
+    println("自定义average函数 : " + myAvg)
+    println("内置的average函数 : " + avg)
+  }
+}

code/spark/spark-core/src/main/java/rdd/scala/SparkSqlTest.scala

@@ -1,57 +0,0 @@
-package rdd.scala
-
-import org.apache.spark.sql.{Dataset, SparkSession}
-
-
-object SparkSqlTest extends App {
-
-
-
-  val spark = SparkSession.builder().appName("Spark SQL basic example").config("spark.some.config.option", "some-value").getOrCreate()
-
-  val dataFrames = spark.read.json("/usr/file/people.json")
-
-  df.select("name").show()
-
-  df.printSchema()
-
-
-
-
-  import spark.implicits._
-
-  val primitiveDS = Seq(1, 2, 3).toDS()
-  primitiveDS.printSchema()
-  primitiveDS.map(_ + 1).collect()
-
-  peopleDS.select("name").show()  //失败
-  peopleDS.dtypes
-  peopleDS.printSchema()
-  peopleDS.toDF()
-  // Encoders are created for case classes
-
-  /* 1.此时把selected写成为selected ,编译器没有任何提示 */
-  spark.sql("selected name from emp")
-
-  /* 2.此时把selected写成为selected ,编译器有提示; 但是把字段名称name写成了nameEd ,编译器没有任何提示*/
-  val dataFrames = spark.read.json("people.json")
-  dataFrames.selected("nameEd").show()
-  dataFrames.map(line=>line.name)
-
-  case class Person(name: String, age: Long)
-
-  /* 3.此时最为严格,语法和字段名称错误都被检测出来*/
-  val dataSet: Dataset[Person] = spark.read.json("people.json").as[Person]
-  dataSet.selected("name")
-  dataSet.map(line=>line.name)
-  dataSet.map(line=>line.nameEd)
-
-  /* 4.即使在由RDD转换为dataFrame时候指定了类型Person,依然无法提示字段名称*/
-  val peopleDF = spark.sparkContext
-    .textFile("people.json")
-    .map(_.split(","))
-    .map(attributes => Person(attributes(0), attributes(1).trim.toInt))
-    .toDF()
-  peopleDF.map(line=>line.name)
-
-}

code/spark/spark-core/src/main/java/rdd/scala/TransformationTest.scala

@@ -67,7 +67,7 @@ class TransformationTest {
   @Test
   def sample(): Unit = {
     val list = List(1, 2, 3, 4, 5, 6)
-    sc.parallelize(list).sample(withReplacement = false, 0.5).foreach(println)
+    sc.parallelize(list).sample(withReplacement = false, fraction = 0.5).foreach(println)
   }