modify
This commit is contained in:
14
code/spark/spark-core/file/emp.json
Normal file
14
code/spark/spark-core/file/emp.json
Normal file
@ -0,0 +1,14 @@
|
||||
{"EMPNO": 7369,"ENAME": "SMITH","JOB": "CLERK","MGR": 7902,"HIREDATE": "1980-12-17 00:00:00","SAL": 800.00,"COMM": null,"DEPTNO": 20}
|
||||
{"EMPNO": 7499,"ENAME": "ALLEN","JOB": "SALESMAN","MGR": 7698,"HIREDATE": "1981-02-20 00:00:00","SAL": 1600.00,"COMM": 300.00,"DEPTNO": 30}
|
||||
{"EMPNO": 7521,"ENAME": "WARD","JOB": "SALESMAN","MGR": 7698,"HIREDATE": "1981-02-22 00:00:00","SAL": 1250.00,"COMM": 500.00,"DEPTNO": 30}
|
||||
{"EMPNO": 7566,"ENAME": "JONES","JOB": "MANAGER","MGR": 7839,"HIREDATE": "1981-04-02 00:00:00","SAL": 2975.00,"COMM": null,"DEPTNO": 20}
|
||||
{"EMPNO": 7654,"ENAME": "MARTIN","JOB": "SALESMAN","MGR": 7698,"HIREDATE": "1981-09-28 00:00:00","SAL": 1250.00,"COMM": 1400.00,"DEPTNO": 30}
|
||||
{"EMPNO": 7698,"ENAME": "BLAKE","JOB": "MANAGER","MGR": 7839,"HIREDATE": "1981-05-01 00:00:00","SAL": 2850.00,"COMM": null,"DEPTNO": 30}
|
||||
{"EMPNO": 7782,"ENAME": "CLARK","JOB": "MANAGER","MGR": 7839,"HIREDATE": "1981-06-09 00:00:00","SAL": 2450.00,"COMM": null,"DEPTNO": 10}
|
||||
{"EMPNO": 7788,"ENAME": "SCOTT","JOB": "ANALYST","MGR": 7566,"HIREDATE": "1987-04-19 00:00:00","SAL": 1500.00,"COMM": null,"DEPTNO": 20}
|
||||
{"EMPNO": 7839,"ENAME": "KING","JOB": "PRESIDENT","MGR": null,"HIREDATE": "1981-11-17 00:00:00","SAL": 5000.00,"COMM": null,"DEPTNO": 10}
|
||||
{"EMPNO": 7844,"ENAME": "TURNER","JOB": "SALESMAN","MGR": 7698,"HIREDATE": "1981-09-08 00:00:00","SAL": 1500.00,"COMM": 0.00,"DEPTNO": 30}
|
||||
{"EMPNO": 7876,"ENAME": "ADAMS","JOB": "CLERK","MGR": 7788,"HIREDATE": "1987-05-23 00:00:00","SAL": 1100.00,"COMM": null,"DEPTNO": 20}
|
||||
{"EMPNO": 7900,"ENAME": "JAMES","JOB": "CLERK","MGR": 7698,"HIREDATE": "1981-12-03 00:00:00","SAL": 950.00,"COMM": null,"DEPTNO": 30}
|
||||
{"EMPNO": 7902,"ENAME": "FORD","JOB": "ANALYST","MGR": 7566,"HIREDATE": "1981-12-03 00:00:00","SAL": 3000.00,"COMM": null,"DEPTNO": 20}
|
||||
{"EMPNO": 7934,"ENAME": "MILLER","JOB": "CLERK","MGR": 7782,"HIREDATE": "1982-01-23 00:00:00","SAL": 1300.00,"COMM": null,"DEPTNO": 10}
|
@ -0,0 +1,65 @@
|
||||
package rdd.scala
|
||||
|
||||
import org.apache.spark.sql.expressions.Aggregator
|
||||
import org.apache.spark.sql.{Encoder, Encoders, SparkSession, functions}
|
||||
|
||||
// 1.定义员工类,对于可能存在null值的字段需要使用Option进行包装
|
||||
case class Emp(ename: String, comm: scala.Option[Double], deptno: Long, empno: Long,
|
||||
hiredate: String, job: String, mgr: scala.Option[Long], sal: Double)
|
||||
|
||||
// 2.定义聚合操作的中间输出类型
|
||||
case class SumAndCount(var sum: Double, var count: Long)
|
||||
|
||||
/* 3.自定义聚合函数
|
||||
* @IN 聚合操作的输入类型
|
||||
* @BUF reduction操作输出值的类型
|
||||
* @OUT 聚合操作的输出类型
|
||||
*/
|
||||
object MyAverage extends Aggregator[Emp, SumAndCount, Double] {
|
||||
|
||||
|
||||
// 4.用于聚合操作的的初始零值
|
||||
override def zero: SumAndCount = SumAndCount(0, 0)
|
||||
|
||||
|
||||
// 5.同一分区中的reduce操作
|
||||
override def reduce(avg: SumAndCount, emp: Emp): SumAndCount = {
|
||||
avg.sum += emp.sal
|
||||
avg.count += 1
|
||||
avg
|
||||
}
|
||||
|
||||
// 6.不同分区中的merge操作
|
||||
override def merge(avg1: SumAndCount, avg2: SumAndCount): SumAndCount = {
|
||||
avg1.sum += avg2.sum
|
||||
avg1.count += avg2.count
|
||||
avg1
|
||||
}
|
||||
|
||||
// 7.定义最终的输出类型
|
||||
override def finish(reduction: SumAndCount): Double = reduction.sum / reduction.count
|
||||
|
||||
// 8.中间类型的编码转换
|
||||
override def bufferEncoder: Encoder[SumAndCount] = Encoders.product
|
||||
|
||||
// 9.输出类型的编码转换
|
||||
override def outputEncoder: Encoder[Double] = Encoders.scalaDouble
|
||||
}
|
||||
|
||||
object SparkSqlApp {
|
||||
|
||||
// 测试方法
|
||||
def main(args: Array[String]): Unit = {
|
||||
|
||||
val spark = SparkSession.builder().appName("Spark-SQL").master("local[2]").getOrCreate()
|
||||
import spark.implicits._
|
||||
val ds = spark.read.json("file/emp.json").as[Emp]
|
||||
|
||||
// 10.使用内置avg()函数和自定义函数分别进行计算,验证自定义函数是否正确
|
||||
val myAvg = ds.select(MyAverage.toColumn.name("average_sal")).first()
|
||||
val avg = ds.select(functions.avg(ds.col("sal"))).first().get(0)
|
||||
|
||||
println("自定义average函数 : " + myAvg)
|
||||
println("内置的average函数 : " + avg)
|
||||
}
|
||||
}
|
@ -1,57 +0,0 @@
|
||||
package rdd.scala
|
||||
|
||||
import org.apache.spark.sql.{Dataset, SparkSession}
|
||||
|
||||
|
||||
object SparkSqlTest extends App {
|
||||
|
||||
|
||||
|
||||
val spark = SparkSession.builder().appName("Spark SQL basic example").config("spark.some.config.option", "some-value").getOrCreate()
|
||||
|
||||
val dataFrames = spark.read.json("/usr/file/people.json")
|
||||
|
||||
df.select("name").show()
|
||||
|
||||
df.printSchema()
|
||||
|
||||
|
||||
|
||||
|
||||
import spark.implicits._
|
||||
|
||||
val primitiveDS = Seq(1, 2, 3).toDS()
|
||||
primitiveDS.printSchema()
|
||||
primitiveDS.map(_ + 1).collect()
|
||||
|
||||
peopleDS.select("name").show() //失败
|
||||
peopleDS.dtypes
|
||||
peopleDS.printSchema()
|
||||
peopleDS.toDF()
|
||||
// Encoders are created for case classes
|
||||
|
||||
/* 1.此时把selected写成为selected ,编译器没有任何提示 */
|
||||
spark.sql("selected name from emp")
|
||||
|
||||
/* 2.此时把selected写成为selected ,编译器有提示; 但是把字段名称name写成了nameEd ,编译器没有任何提示*/
|
||||
val dataFrames = spark.read.json("people.json")
|
||||
dataFrames.selected("nameEd").show()
|
||||
dataFrames.map(line=>line.name)
|
||||
|
||||
case class Person(name: String, age: Long)
|
||||
|
||||
/* 3.此时最为严格,语法和字段名称错误都被检测出来*/
|
||||
val dataSet: Dataset[Person] = spark.read.json("people.json").as[Person]
|
||||
dataSet.selected("name")
|
||||
dataSet.map(line=>line.name)
|
||||
dataSet.map(line=>line.nameEd)
|
||||
|
||||
/* 4.即使在由RDD转换为dataFrame时候指定了类型Person,依然无法提示字段名称*/
|
||||
val peopleDF = spark.sparkContext
|
||||
.textFile("people.json")
|
||||
.map(_.split(","))
|
||||
.map(attributes => Person(attributes(0), attributes(1).trim.toInt))
|
||||
.toDF()
|
||||
peopleDF.map(line=>line.name)
|
||||
|
||||
}
|
@ -67,7 +67,7 @@ class TransformationTest {
|
||||
@Test
|
||||
def sample(): Unit = {
|
||||
val list = List(1, 2, 3, 4, 5, 6)
|
||||
sc.parallelize(list).sample(withReplacement = false, 0.5).foreach(println)
|
||||
sc.parallelize(list).sample(withReplacement = false, fraction = 0.5).foreach(println)
|
||||
}
|
||||
|
||||
|
||||
|
Reference in New Issue
Block a user