From 3af833743fd416612dc9a567ec927fb93b6d7ec7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E7=BD=97=E7=A5=A5?= <1366971433@qq.com> Date: Fri, 5 Apr 2019 08:45:16 +0800 Subject: [PATCH] spring/spring boot+mybatis+phoenix --- .gitignore | 1 - .../spring-boot-mybatis-phoenix/pom.xml | 56 +++++++++++++ .../SpringBootMybatisApplication.java | 14 ++++ .../springboot/bean/USPopulation.java | 18 +++++ .../springboot/dao/PopulationDao.java | 24 ++++++ .../src/main/resources/application.yml | 32 ++++++++ .../heibaiying/springboot/PopulationTest.java | 53 +++++++++++++ code/Phoenix/spring-mybatis-phoenix/pom.xml | 74 ++++++++++++++++++ .../com/heibaiying/bean/USPopulation.java | 15 ++++ .../com/heibaiying/dao/PopulationDao.java | 17 ++++ .../src/main/resources/jdbc.properties | 4 + .../src/main/resources/mappers/Population.xml | 24 ++++++ .../src/main/resources/mybatisConfig.xml | 17 ++++ .../src/main/resources/springApplication.xml | 38 +++++++++ .../com/heibaiying/dao/PopulationDaoTest.java | 50 ++++++++++++ code/spark/spark-base/input/wc.txt | 2 + .../spark-base/output/wcResult/._SUCCESS.crc | Bin 0 -> 8 bytes .../output/wcResult/.part-00000.crc | Bin 0 -> 12 bytes .../output/wcResult/.part-00001.crc | Bin 0 -> 12 bytes .../spark/spark-base/output/wcResult/_SUCCESS | 0 .../spark-base/output/wcResult/part-00000 | 1 + .../spark-base/output/wcResult/part-00001 | 2 + code/spark/spark-base/pom.xml | 46 +++++++++++ .../com/heibaiying/spark/rdd/WordCount.scala | 16 ++++ 24 files changed, 503 insertions(+), 1 deletion(-) create mode 100644 code/Phoenix/spring-boot-mybatis-phoenix/pom.xml create mode 100644 code/Phoenix/spring-boot-mybatis-phoenix/src/main/java/com/heibaiying/springboot/SpringBootMybatisApplication.java create mode 100644 code/Phoenix/spring-boot-mybatis-phoenix/src/main/java/com/heibaiying/springboot/bean/USPopulation.java create mode 100644 code/Phoenix/spring-boot-mybatis-phoenix/src/main/java/com/heibaiying/springboot/dao/PopulationDao.java create mode 100644 code/Phoenix/spring-boot-mybatis-phoenix/src/main/resources/application.yml create mode 100644 code/Phoenix/spring-boot-mybatis-phoenix/src/test/java/com/heibaiying/springboot/PopulationTest.java create mode 100644 code/Phoenix/spring-mybatis-phoenix/pom.xml create mode 100644 code/Phoenix/spring-mybatis-phoenix/src/main/java/com/heibaiying/bean/USPopulation.java create mode 100644 code/Phoenix/spring-mybatis-phoenix/src/main/java/com/heibaiying/dao/PopulationDao.java create mode 100644 code/Phoenix/spring-mybatis-phoenix/src/main/resources/jdbc.properties create mode 100644 code/Phoenix/spring-mybatis-phoenix/src/main/resources/mappers/Population.xml create mode 100644 code/Phoenix/spring-mybatis-phoenix/src/main/resources/mybatisConfig.xml create mode 100644 code/Phoenix/spring-mybatis-phoenix/src/main/resources/springApplication.xml create mode 100644 code/Phoenix/spring-mybatis-phoenix/src/test/java/com/heibaiying/dao/PopulationDaoTest.java create mode 100644 code/spark/spark-base/input/wc.txt create mode 100644 code/spark/spark-base/output/wcResult/._SUCCESS.crc create mode 100644 code/spark/spark-base/output/wcResult/.part-00000.crc create mode 100644 code/spark/spark-base/output/wcResult/.part-00001.crc create mode 100644 code/spark/spark-base/output/wcResult/_SUCCESS create mode 100644 code/spark/spark-base/output/wcResult/part-00000 create mode 100644 code/spark/spark-base/output/wcResult/part-00001 create mode 100644 code/spark/spark-base/pom.xml create mode 100644 code/spark/spark-base/src/main/scala/com/heibaiying/spark/rdd/WordCount.scala diff --git a/.gitignore b/.gitignore index 23646f8..c3f7281 100644 --- a/.gitignore +++ b/.gitignore @@ -18,7 +18,6 @@ .settings .springBeans /build -/code MANIFEST.MF _site/ activemq-data diff --git a/code/Phoenix/spring-boot-mybatis-phoenix/pom.xml b/code/Phoenix/spring-boot-mybatis-phoenix/pom.xml new file mode 100644 index 0000000..323b837 --- /dev/null +++ b/code/Phoenix/spring-boot-mybatis-phoenix/pom.xml @@ -0,0 +1,56 @@ + + + 4.0.0 + + org.springframework.boot + spring-boot-starter-parent + 2.1.4.RELEASE + + + com.heibaiying + spring-boot-mybatis-phoenix + 0.0.1-SNAPSHOT + spring-boot-mybatis-phoenix + mybatis project for Spring Boot + + + 1.8 + + + + + + org.mybatis.spring.boot + mybatis-spring-boot-starter + 1.3.2 + + + + org.apache.phoenix + phoenix-core + 4.14.0-cdh5.14.2 + + + org.projectlombok + lombok + true + + + org.springframework.boot + spring-boot-starter-test + test + + + + + + + org.springframework.boot + spring-boot-maven-plugin + + + + + diff --git a/code/Phoenix/spring-boot-mybatis-phoenix/src/main/java/com/heibaiying/springboot/SpringBootMybatisApplication.java b/code/Phoenix/spring-boot-mybatis-phoenix/src/main/java/com/heibaiying/springboot/SpringBootMybatisApplication.java new file mode 100644 index 0000000..21da69f --- /dev/null +++ b/code/Phoenix/spring-boot-mybatis-phoenix/src/main/java/com/heibaiying/springboot/SpringBootMybatisApplication.java @@ -0,0 +1,14 @@ +package com.heibaiying.springboot; + +import org.springframework.boot.SpringApplication; +import org.springframework.boot.autoconfigure.SpringBootApplication; + +@SpringBootApplication +public class SpringBootMybatisApplication { + + public static void main(String[] args) { + SpringApplication.run(SpringBootMybatisApplication.class, args); + } + +} + diff --git a/code/Phoenix/spring-boot-mybatis-phoenix/src/main/java/com/heibaiying/springboot/bean/USPopulation.java b/code/Phoenix/spring-boot-mybatis-phoenix/src/main/java/com/heibaiying/springboot/bean/USPopulation.java new file mode 100644 index 0000000..f5502fb --- /dev/null +++ b/code/Phoenix/spring-boot-mybatis-phoenix/src/main/java/com/heibaiying/springboot/bean/USPopulation.java @@ -0,0 +1,18 @@ +package com.heibaiying.springboot.bean; + +import lombok.AllArgsConstructor; +import lombok.Data; +import lombok.NoArgsConstructor; +import lombok.ToString; + +@Data +@AllArgsConstructor +@NoArgsConstructor +@ToString +public class USPopulation { + + private String state; + private String city; + private long population; + +} diff --git a/code/Phoenix/spring-boot-mybatis-phoenix/src/main/java/com/heibaiying/springboot/dao/PopulationDao.java b/code/Phoenix/spring-boot-mybatis-phoenix/src/main/java/com/heibaiying/springboot/dao/PopulationDao.java new file mode 100644 index 0000000..bb94a7d --- /dev/null +++ b/code/Phoenix/spring-boot-mybatis-phoenix/src/main/java/com/heibaiying/springboot/dao/PopulationDao.java @@ -0,0 +1,24 @@ +package com.heibaiying.springboot.dao; + +import com.heibaiying.springboot.bean.USPopulation; +import org.apache.ibatis.annotations.*; + +import java.util.List; + +@Mapper +public interface PopulationDao { + + @Select("SELECT * from us_population") + List queryAll(); + + @Insert("UPSERT INTO us_population VALUES( #{state}, #{city}, #{population} )") + void save(USPopulation USPopulation); + + @Select("SELECT * FROM us_population WHERE state=#{state} AND city = #{city}") + USPopulation queryByStateAndCity(String state, String city); + + + @Delete("DELETE FROM us_population WHERE state=#{state} AND city = #{city}") + void deleteByStateAndCity(String state, String city); + +} diff --git a/code/Phoenix/spring-boot-mybatis-phoenix/src/main/resources/application.yml b/code/Phoenix/spring-boot-mybatis-phoenix/src/main/resources/application.yml new file mode 100644 index 0000000..627fc0e --- /dev/null +++ b/code/Phoenix/spring-boot-mybatis-phoenix/src/main/resources/application.yml @@ -0,0 +1,32 @@ +spring: + datasource: + #zookeeper地址 + url: jdbc:phoenix:192.168.0.105:2181 + driver-class-name: org.apache.phoenix.jdbc.PhoenixDriver + + # 如果不想配置对数据库连接池做特殊配置的话,以下关于连接池的配置就不是必须的 + # spring-boot 2.X 默认采用高性能的 Hikari 作为连接池 更多配置可以参考 https://github.com/brettwooldridge/HikariCP#configuration-knobs-baby + type: com.zaxxer.hikari.HikariDataSource + hikari: + # 池中维护的最小空闲连接数 + minimum-idle: 10 + # 池中最大连接数，包括闲置和使用中的连接 + maximum-pool-size: 20 + # 此属性控制从池返回的连接的默认自动提交行为。默认为true + auto-commit: true + # 允许最长空闲时间 + idle-timeout: 30000 + # 此属性表示连接池的用户定义名称，主要显示在日志记录和JMX管理控制台中，以标识池和池配置。 默认值：自动生成 + pool-name: custom-hikari + #此属性控制池中连接的最长生命周期，值0表示无限生命周期，默认1800000即30分钟 + max-lifetime: 1800000 + # 数据库连接超时时间,默认30秒，即30000 + connection-timeout: 30000 + # 连接测试sql 这个地方需要根据数据库方言差异而配置 例如 oracle 就应该写成 select 1 from dual + connection-test-query: SELECT 1 + +# mybatis 相关配置 +mybatis: + configuration: + # 是否打印sql语句 调试的时候可以开启 + log-impl: org.apache.ibatis.logging.stdout.StdOutImpl \ No newline at end of file diff --git a/code/Phoenix/spring-boot-mybatis-phoenix/src/test/java/com/heibaiying/springboot/PopulationTest.java b/code/Phoenix/spring-boot-mybatis-phoenix/src/test/java/com/heibaiying/springboot/PopulationTest.java new file mode 100644 index 0000000..2d9b7dd --- /dev/null +++ b/code/Phoenix/spring-boot-mybatis-phoenix/src/test/java/com/heibaiying/springboot/PopulationTest.java @@ -0,0 +1,53 @@ +package com.heibaiying.springboot; + +import com.heibaiying.springboot.bean.USPopulation; +import com.heibaiying.springboot.dao.PopulationDao; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.boot.test.context.SpringBootTest; +import org.springframework.test.context.junit4.SpringRunner; + +import java.util.List; + +@RunWith(SpringRunner.class) +@SpringBootTest +public class PopulationTest { + + @Autowired + private PopulationDao populationDao; + + @Test + public void queryAll() { + List USPopulationList = populationDao.queryAll(); + if (USPopulationList != null) { + for (USPopulation USPopulation : USPopulationList) { + System.out.println(USPopulation.getCity() + " " + USPopulation.getPopulation()); + } + } + } + + @Test + public void save() { + populationDao.save(new USPopulation("TX", "Dallas", 66666)); + USPopulation usPopulation = populationDao.queryByStateAndCity("TX", "Dallas"); + System.out.println(usPopulation); + } + + @Test + public void update() { + populationDao.save(new USPopulation("TX", "Dallas", 99999)); + USPopulation usPopulation = populationDao.queryByStateAndCity("TX", "Dallas"); + System.out.println(usPopulation); + } + + + @Test + public void delete() { + populationDao.deleteByStateAndCity("TX", "Dallas"); + USPopulation usPopulation = populationDao.queryByStateAndCity("TX", "Dallas"); + System.out.println(usPopulation); + } + +} + diff --git a/code/Phoenix/spring-mybatis-phoenix/pom.xml b/code/Phoenix/spring-mybatis-phoenix/pom.xml new file mode 100644 index 0000000..7b0d45b --- /dev/null +++ b/code/Phoenix/spring-mybatis-phoenix/pom.xml @@ -0,0 +1,74 @@ + + + 4.0.0 + + com.heibaiying + spring-mybatis-phoenix + 1.0-SNAPSHOT + + 5.1.6.RELEASE + + + + + org.springframework + spring-context + ${spring-base-version} + + + org.springframework + spring-beans + ${spring-base-version} + + + org.springframework + spring-core + ${spring-base-version} + + + org.projectlombok + lombok + 1.18.4 + provided + + + + org.springframework + spring-jdbc + ${spring-base-version} + + + + junit + junit + 4.12 + test + + + org.springframework + spring-test + ${spring-base-version} + test + + + + org.mybatis + mybatis-spring + 1.3.2 + + + org.mybatis + mybatis + 3.4.6 + + + + org.apache.phoenix + phoenix-core + 4.14.0-cdh5.14.2 + + + + \ No newline at end of file diff --git a/code/Phoenix/spring-mybatis-phoenix/src/main/java/com/heibaiying/bean/USPopulation.java b/code/Phoenix/spring-mybatis-phoenix/src/main/java/com/heibaiying/bean/USPopulation.java new file mode 100644 index 0000000..ad67a51 --- /dev/null +++ b/code/Phoenix/spring-mybatis-phoenix/src/main/java/com/heibaiying/bean/USPopulation.java @@ -0,0 +1,15 @@ +package com.heibaiying.bean; + +import lombok.AllArgsConstructor; +import lombok.Data; +import lombok.NoArgsConstructor; + +@Data +@AllArgsConstructor +@NoArgsConstructor +public class USPopulation { + + private String state; + private String city; + private long population; +} diff --git a/code/Phoenix/spring-mybatis-phoenix/src/main/java/com/heibaiying/dao/PopulationDao.java b/code/Phoenix/spring-mybatis-phoenix/src/main/java/com/heibaiying/dao/PopulationDao.java new file mode 100644 index 0000000..18fa69b --- /dev/null +++ b/code/Phoenix/spring-mybatis-phoenix/src/main/java/com/heibaiying/dao/PopulationDao.java @@ -0,0 +1,17 @@ +package com.heibaiying.dao; + +import com.heibaiying.bean.USPopulation; +import org.apache.ibatis.annotations.Param; + +import java.util.List; + +public interface PopulationDao { + + List queryAll(); + + void save(USPopulation USPopulation); + + USPopulation queryByStateAndCity(@Param("state") String state, @Param("city") String city); + + void deleteByStateAndCity(@Param("state") String state, @Param("city") String city); +} diff --git a/code/Phoenix/spring-mybatis-phoenix/src/main/resources/jdbc.properties b/code/Phoenix/spring-mybatis-phoenix/src/main/resources/jdbc.properties new file mode 100644 index 0000000..91a00da --- /dev/null +++ b/code/Phoenix/spring-mybatis-phoenix/src/main/resources/jdbc.properties @@ -0,0 +1,4 @@ +# ݿ +phoenix.driverClassName=org.apache.phoenix.jdbc.PhoenixDriver +# zookeeperַ +phoenix.url=jdbc:phoenix:192.168.0.105:2181 diff --git a/code/Phoenix/spring-mybatis-phoenix/src/main/resources/mappers/Population.xml b/code/Phoenix/spring-mybatis-phoenix/src/main/resources/mappers/Population.xml new file mode 100644 index 0000000..26e05fb --- /dev/null +++ b/code/Phoenix/spring-mybatis-phoenix/src/main/resources/mappers/Population.xml @@ -0,0 +1,24 @@ + + + + + + + + + UPSERT INTO us_population VALUES( #{state}, #{city}, #{population} ) + + + + + + DELETE FROM us_population WHERE state=#{state} AND city = #{city} + + + \ No newline at end of file diff --git a/code/Phoenix/spring-mybatis-phoenix/src/main/resources/mybatisConfig.xml b/code/Phoenix/spring-mybatis-phoenix/src/main/resources/mybatisConfig.xml new file mode 100644 index 0000000..d60965c --- /dev/null +++ b/code/Phoenix/spring-mybatis-phoenix/src/main/resources/mybatisConfig.xml @@ -0,0 +1,17 @@ + + + + + + + + + + + + + + + diff --git a/code/Phoenix/spring-mybatis-phoenix/src/main/resources/springApplication.xml b/code/Phoenix/spring-mybatis-phoenix/src/main/resources/springApplication.xml new file mode 100644 index 0000000..ba548c7 --- /dev/null +++ b/code/Phoenix/spring-mybatis-phoenix/src/main/resources/springApplication.xml @@ -0,0 +1,38 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/code/Phoenix/spring-mybatis-phoenix/src/test/java/com/heibaiying/dao/PopulationDaoTest.java b/code/Phoenix/spring-mybatis-phoenix/src/test/java/com/heibaiying/dao/PopulationDaoTest.java new file mode 100644 index 0000000..90d02c0 --- /dev/null +++ b/code/Phoenix/spring-mybatis-phoenix/src/test/java/com/heibaiying/dao/PopulationDaoTest.java @@ -0,0 +1,50 @@ +package com.heibaiying.dao; + +import com.heibaiying.bean.USPopulation; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.test.context.ContextConfiguration; +import org.springframework.test.context.junit4.SpringRunner; + +import java.util.List; + +@RunWith(SpringRunner.class) +@ContextConfiguration({"classpath:springApplication.xml"}) +public class PopulationDaoTest { + + @Autowired + private PopulationDao populationDao; + + @Test + public void queryAll() { + List USPopulationList = populationDao.queryAll(); + if (USPopulationList != null) { + for (USPopulation USPopulation : USPopulationList) { + System.out.println(USPopulation.getCity() + " " + USPopulation.getPopulation()); + } + } + } + + @Test + public void save() { + populationDao.save(new USPopulation("TX", "Dallas", 66666)); + USPopulation usPopulation = populationDao.queryByStateAndCity("TX", "Dallas"); + System.out.println(usPopulation); + } + + @Test + public void update() { + populationDao.save(new USPopulation("TX", "Dallas", 99999)); + USPopulation usPopulation = populationDao.queryByStateAndCity("TX", "Dallas"); + System.out.println(usPopulation); + } + + + @Test + public void delete() { + populationDao.deleteByStateAndCity("TX", "Dallas"); + USPopulation usPopulation = populationDao.queryByStateAndCity("TX", "Dallas"); + System.out.println(usPopulation); + } +} diff --git a/code/spark/spark-base/input/wc.txt b/code/spark/spark-base/input/wc.txt new file mode 100644 index 0000000..b97f2c6 --- /dev/null +++ b/code/spark/spark-base/input/wc.txt @@ -0,0 +1,2 @@ +hadoop,mapreduce,hadoop +spark,spark \ No newline at end of file diff --git a/code/spark/spark-base/output/wcResult/._SUCCESS.crc b/code/spark/spark-base/output/wcResult/._SUCCESS.crc new file mode 100644 index 0000000000000000000000000000000000000000..3b7b044936a890cd8d651d349a752d819d71d22c GIT binary patch literal 8 PcmYc;N@ieSU}69O2$TUk literal 0 HcmV?d00001 diff --git a/code/spark/spark-base/output/wcResult/.part-00000.crc b/code/spark/spark-base/output/wcResult/.part-00000.crc new file mode 100644 index 0000000000000000000000000000000000000000..c9619cf94dd6878c8290c8959cfc002192bbb1d2 GIT binary patch literal 12 TcmYc;N@ieSU}BgUS + + 4.0.0 + + com.heibaiying + spark-base + 1.0-SNAPSHOT + + 2.12.8 + + + + + + org.apache.maven.plugins + maven-compiler-plugin + + 8 + 8 + + + + + + + + + org.scala-lang + scala-library + ${scala.version} + + + org.apache.spark + spark-core_2.12 + 2.4.0 + + + com.thoughtworks.paranamer + paranamer + 2.8 + + + + \ No newline at end of file diff --git a/code/spark/spark-base/src/main/scala/com/heibaiying/spark/rdd/WordCount.scala b/code/spark/spark-base/src/main/scala/com/heibaiying/spark/rdd/WordCount.scala new file mode 100644 index 0000000..bd418b7 --- /dev/null +++ b/code/spark/spark-base/src/main/scala/com/heibaiying/spark/rdd/WordCount.scala @@ -0,0 +1,16 @@ +package com.heibaiying.spark.rdd + +import org.apache.spark.{SparkConf, SparkContext} + + +object WordCount { + + def main(args: Array[String]): Unit = { + val conf = new SparkConf().setAppName("sparkBase").setMaster("local[2]") + val sc = new SparkContext(conf) + val rdd = sc.textFile("input/wc.txt").flatMap(_.split(",")).map((_, 1)).reduceByKey(_ + _) + rdd.foreach(println) + rdd.saveAsTextFile("output/") + } + +} \ No newline at end of file