strom集成其他框架

code/Storm/storm-hdfs-integration/src/main/java/com.heibaiying/DataToHdfsApp.java

@@ -20,18 +20,19 @@ import org.apache.storm.hdfs.bolt.sync.SyncPolicy;
 import org.apache.storm.topology.TopologyBuilder;
 
 /**
- * 进行词频统计 并将统计结果存储到HDFS中
- * <p>
- * hdfs://hadoopp001:8020 path
+ * 将样本数据存储到HDFS中
  */
-public class WordCountToHdfsApp {
+public class DataToHdfsApp {
 
     private static final String DATA_SOURCE_SPOUT = "dataSourceSpout";
     private static final String HDFS_BOLT = "hdfsBolt";
 
     public static void main(String[] args) {
 
-        // 定义存储文本的分隔符
+        // 指定Hadoop的用户名 如果不指定,则在HDFS创建目录时候有可能抛出无权限的异常(RemoteException: Permission denied)
+        System.setProperty("HADOOP_USER_NAME", "root");
+
+        // 定义输出字段(Field)之间的分隔符
         RecordFormat format = new DelimitedRecordFormat()
                 .withFieldDelimiter("|");
 
@@ -41,7 +42,7 @@ public class WordCountToHdfsApp {
         // 文件策略: 每个文件大小上限1M,超过限定时,创建新文件并继续写入
         FileRotationPolicy rotationPolicy = new FileSizeRotationPolicy(1.0f, Units.MB);
 
-        // 定义完整路径
+        // 定义存储路径
         FileNameFormat fileNameFormat = new DefaultFileNameFormat()
                 .withPath("/storm-hdfs/");
 
@@ -57,20 +58,20 @@ public class WordCountToHdfsApp {
         // 构建Topology
         TopologyBuilder builder = new TopologyBuilder();
         builder.setSpout(DATA_SOURCE_SPOUT, new DataSourceSpout());
-        // save to HBase
+        // save to HDFS
         builder.setBolt(HDFS_BOLT, hdfsBolt, 1).shuffleGrouping(DATA_SOURCE_SPOUT);
 
 
         // 如果外部传参cluster则代表线上环境启动,否则代表本地启动
         if (args.length > 0 && args[0].equals("cluster")) {
             try {
-                StormSubmitter.submitTopology("ClusterWordCountToHdfsApp", new Config(), builder.createTopology());
+                StormSubmitter.submitTopology("ClusterDataToHdfsApp", new Config(), builder.createTopology());
             } catch (AlreadyAliveException | InvalidTopologyException | AuthorizationException e) {
                 e.printStackTrace();
             }
         } else {
             LocalCluster cluster = new LocalCluster();
-            cluster.submitTopology("LocalWordCountToHdfsApp",
+            cluster.submitTopology("LocalDataToHdfsApp",
                     new Config(), builder.createTopology());
         }
     }