blog/BigData-Notes
2
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

strom整合其他框架

Browse Source
This commit is contained in:
罗祥
2019-04-17 17:55:19 +08:00
parent a339956c1a
commit d4daa775cf
13 changed files with 517 additions and 14 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
code/Storm/storm-redis-integration/pom.xml
View File

@ -10,7 +10,7 @@
<properties>
<project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
<storm.version>1.2.0</storm.version>
<storm.version>1.2.2</storm.version>
</properties>
<build>

12
code/Storm/storm-redis-integration/src/main/java/com/heibaiying/CustomRedisCountApp.java
View File

@ -18,22 +18,26 @@ import org.apache.storm.topology.TopologyBuilder;
*/
public class CustomRedisCountApp {
private static final String DATA_SOURCE_SPOUT = "dataSourceSpout";
private static final String SPLIT_BOLT = "splitBolt";
private static final String STORE_BOLT = "storeBolt";
private static final String REDIS_HOST = "192.168.200.226";
private static final int REDIS_PORT = 6379;
public static void main(String[] args) {
TopologyBuilder builder = new TopologyBuilder();
builder.setSpout("dataSourceSpout", new DataSourceSpout());
builder.setSpout(DATA_SOURCE_SPOUT, new DataSourceSpout());
// split
builder.setBolt("splitBolt", new SplitBolt()).shuffleGrouping("dataSourceSpout");
builder.setBolt(SPLIT_BOLT, new SplitBolt()).shuffleGrouping(DATA_SOURCE_SPOUT);
// save to redis and count
JedisPoolConfig poolConfig = new JedisPoolConfig.Builder()
.setHost(REDIS_HOST).setPort(REDIS_PORT).build();
RedisStoreMapper storeMapper = new WordCountStoreMapper();
RedisCountStoreBolt countStoreBolt = new RedisCountStoreBolt(poolConfig, storeMapper);
builder.setBolt("storeBolt", countStoreBolt).shuffleGrouping("splitBolt");
builder.setBolt(STORE_BOLT, countStoreBolt).shuffleGrouping(SPLIT_BOLT);
// 如果外部传参cluster则代表线上环境启动否则代表本地启动
// 如果外部传参cluster则代表线上环境启动,否则代表本地启动
if (args.length > 0 && args[0].equals("cluster")) {
try {
StormSubmitter.submitTopology("ClusterCustomRedisCountApp", new Config(), builder.createTopology());

18
code/Storm/storm-redis-integration/src/main/java/com/heibaiying/WordCountToRedisApp.java
View File

@ -17,29 +17,35 @@ import org.apache.storm.topology.TopologyBuilder;
/**
* 进行词频统计 并将统计结果存储到Redis中
*
* <p>
* 编译打包: mvn clean assembly:assembly -Dmaven.test.skip=true
* 提交Topology到集群: storm jar /usr/appjar/storm-redis-integration-1.0-jar-with-dependencies.jar com.heibaiying.WordCountToRedisApp cluster
* 提交Topology到集群: storm jar /usr/appjar/storm-redis-integration-1.0-with-dependencies.jar com.heibaiying.WordCountToRedisApp cluster
* 停止Topology: storm kill ClusterWordCountApp -w 3
*/
public class WordCountToRedisApp {
private static final String DATA_SOURCE_SPOUT = "dataSourceSpout";
private static final String SPLIT_BOLT = "splitBolt";
private static final String COUNT_BOLT = "countBolt";
private static final String STORE_BOLT = "storeBolt";
//在实际开发中这些参数可以将通过外部传入 使得程序更加灵活
private static final String REDIS_HOST = "192.168.200.226";
private static final int REDIS_PORT = 6379;
public static void main(String[] args) {
TopologyBuilder builder = new TopologyBuilder();
builder.setSpout("dataSourceSpout", new DataSourceSpout());
builder.setSpout(DATA_SOURCE_SPOUT, new DataSourceSpout());
// split
builder.setBolt("splitBolt", new SplitBolt()).shuffleGrouping("dataSourceSpout");
builder.setBolt(SPLIT_BOLT, new SplitBolt()).shuffleGrouping(DATA_SOURCE_SPOUT);
// count
builder.setBolt("countBolt", new CountBolt()).shuffleGrouping("splitBolt");
builder.setBolt(COUNT_BOLT, new CountBolt()).shuffleGrouping(SPLIT_BOLT);
// save to redis
JedisPoolConfig poolConfig = new JedisPoolConfig.Builder()
.setHost(REDIS_HOST).setPort(REDIS_PORT).build();
RedisStoreMapper storeMapper = new WordCountStoreMapper();
RedisStoreBolt storeBolt = new RedisStoreBolt(poolConfig, storeMapper);
builder.setBolt("storeBolt", storeBolt).shuffleGrouping("countBolt");
builder.setBolt(STORE_BOLT, storeBolt).shuffleGrouping(COUNT_BOLT);
// 如果外部传参cluster则代表线上环境启动否则代表本地启动
if (args.length > 0 && args[0].equals("cluster")) {

2
code/Storm/storm-redis-integration/src/main/resources/assembly.xml
View File

@ -2,7 +2,7 @@
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xsi:schemaLocation="http://maven.apache.org/ASSEMBLY/2.0.0 http://maven.apache.org/xsd/assembly-2.0.0.xsd">
<id>jar-with-dependencies</id>
<id>with-dependencies</id>
<!--指明打包方式-->
<formats>