flink状态管理
This commit is contained in:
罗祥
parent
088d20afb0
commit
0d5a81bc7c
237
code/Flink/flink-state-management/pom.xml
Normal file
237
code/Flink/flink-state-management/pom.xml
Normal file
@ -0,0 +1,237 @@
|
|||||||
|
<!--
|
||||||
|
Licensed to the Apache Software Foundation (ASF) under one
|
||||||
|
or more contributor license agreements. See the NOTICE file
|
||||||
|
distributed with this work for additional information
|
||||||
|
regarding copyright ownership. The ASF licenses this file
|
||||||
|
to you under the Apache License, Version 2.0 (the
|
||||||
|
"License"); you may not use this file except in compliance
|
||||||
|
with the License. You may obtain a copy of the License at
|
||||||
|
|
||||||
|
http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
|
||||||
|
Unless required by applicable law or agreed to in writing,
|
||||||
|
software distributed under the License is distributed on an
|
||||||
|
"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
||||||
|
KIND, either express or implied. See the License for the
|
||||||
|
specific language governing permissions and limitations
|
||||||
|
under the License.
|
||||||
|
-->
|
||||||
|
<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
|
||||||
|
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
|
||||||
|
<modelVersion>4.0.0</modelVersion>
|
||||||
|
|
||||||
|
<groupId>com.heibaiying</groupId>
|
||||||
|
<artifactId>flink-state-management</artifactId>
|
||||||
|
<version>1.0</version>
|
||||||
|
<packaging>jar</packaging>
|
||||||
|
|
||||||
|
<name>Flink Quickstart Job</name>
|
||||||
|
<url>http://www.myorganization.org</url>
|
||||||
|
|
||||||
|
<properties>
|
||||||
|
<project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
|
||||||
|
<flink.version>1.9.0</flink.version>
|
||||||
|
<java.version>1.8</java.version>
|
||||||
|
<scala.binary.version>2.11</scala.binary.version>
|
||||||
|
<maven.compiler.source>${java.version}</maven.compiler.source>
|
||||||
|
<maven.compiler.target>${java.version}</maven.compiler.target>
|
||||||
|
</properties>
|
||||||
|
|
||||||
|
<repositories>
|
||||||
|
<repository>
|
||||||
|
<id>apache.snapshots</id>
|
||||||
|
<name>Apache Development Snapshot Repository</name>
|
||||||
|
<url>https://repository.apache.org/content/repositories/snapshots/</url>
|
||||||
|
<releases>
|
||||||
|
<enabled>false</enabled>
|
||||||
|
</releases>
|
||||||
|
<snapshots>
|
||||||
|
<enabled>true</enabled>
|
||||||
|
</snapshots>
|
||||||
|
</repository>
|
||||||
|
</repositories>
|
||||||
|
|
||||||
|
<dependencies>
|
||||||
|
<!-- Apache Flink dependencies -->
|
||||||
|
<!-- These dependencies are provided, because they should not be packaged into the JAR file. -->
|
||||||
|
<dependency>
|
||||||
|
<groupId>org.apache.flink</groupId>
|
||||||
|
<artifactId>flink-java</artifactId>
|
||||||
|
<version>${flink.version}</version>
|
||||||
|
<scope>provided</scope>
|
||||||
|
</dependency>
|
||||||
|
<dependency>
|
||||||
|
<groupId>org.apache.flink</groupId>
|
||||||
|
<artifactId>flink-streaming-java_${scala.binary.version}</artifactId>
|
||||||
|
<version>${flink.version}</version>
|
||||||
|
<scope>provided</scope>
|
||||||
|
</dependency>
|
||||||
|
|
||||||
|
<!-- Add connector dependencies here. They must be in the default scope (compile). -->
|
||||||
|
|
||||||
|
<!-- Example:
|
||||||
|
|
||||||
|
<dependency>
|
||||||
|
<groupId>org.apache.flink</groupId>
|
||||||
|
<artifactId>flink-connector-kafka-0.10_${scala.binary.version}</artifactId>
|
||||||
|
<version>${flink.version}</version>
|
||||||
|
</dependency>
|
||||||
|
-->
|
||||||
|
|
||||||
|
<!-- Add logging framework, to produce console output when running in the IDE. -->
|
||||||
|
<!-- These dependencies are excluded from the application JAR by default. -->
|
||||||
|
<dependency>
|
||||||
|
<groupId>org.slf4j</groupId>
|
||||||
|
<artifactId>slf4j-log4j12</artifactId>
|
||||||
|
<version>1.7.7</version>
|
||||||
|
<scope>runtime</scope>
|
||||||
|
</dependency>
|
||||||
|
<dependency>
|
||||||
|
<groupId>log4j</groupId>
|
||||||
|
<artifactId>log4j</artifactId>
|
||||||
|
<version>1.2.17</version>
|
||||||
|
<scope>runtime</scope>
|
||||||
|
</dependency>
|
||||||
|
<dependency>
|
||||||
|
<groupId>org.apache.flink</groupId>
|
||||||
|
<artifactId>flink-statebackend-rocksdb_2.11</artifactId>
|
||||||
|
<version>1.9.0</version>
|
||||||
|
</dependency>
|
||||||
|
</dependencies>
|
||||||
|
|
||||||
|
<build>
|
||||||
|
<plugins>
|
||||||
|
|
||||||
|
<!-- Java Compiler -->
|
||||||
|
<plugin>
|
||||||
|
<groupId>org.apache.maven.plugins</groupId>
|
||||||
|
<artifactId>maven-compiler-plugin</artifactId>
|
||||||
|
<version>3.1</version>
|
||||||
|
<configuration>
|
||||||
|
<source>${java.version}</source>
|
||||||
|
<target>${java.version}</target>
|
||||||
|
</configuration>
|
||||||
|
</plugin>
|
||||||
|
|
||||||
|
<!-- We use the maven-shade plugin to create a fat jar that contains all necessary dependencies. -->
|
||||||
|
<!-- Change the value of <mainClass>...</mainClass> if your program entry point changes. -->
|
||||||
|
<plugin>
|
||||||
|
<groupId>org.apache.maven.plugins</groupId>
|
||||||
|
<artifactId>maven-shade-plugin</artifactId>
|
||||||
|
<version>3.0.0</version>
|
||||||
|
<executions>
|
||||||
|
<!-- Run shade goal on package phase -->
|
||||||
|
<execution>
|
||||||
|
<phase>package</phase>
|
||||||
|
<goals>
|
||||||
|
<goal>shade</goal>
|
||||||
|
</goals>
|
||||||
|
<configuration>
|
||||||
|
<artifactSet>
|
||||||
|
<excludes>
|
||||||
|
<exclude>org.apache.flink:force-shading</exclude>
|
||||||
|
<exclude>com.google.code.findbugs:jsr305</exclude>
|
||||||
|
<exclude>org.slf4j:*</exclude>
|
||||||
|
<exclude>log4j:*</exclude>
|
||||||
|
</excludes>
|
||||||
|
</artifactSet>
|
||||||
|
<filters>
|
||||||
|
<filter>
|
||||||
|
<!-- Do not copy the signatures in the META-INF folder.
|
||||||
|
Otherwise, this might cause SecurityExceptions when using the JAR. -->
|
||||||
|
<artifact>*:*</artifact>
|
||||||
|
<excludes>
|
||||||
|
<exclude>META-INF/*.SF</exclude>
|
||||||
|
<exclude>META-INF/*.DSA</exclude>
|
||||||
|
<exclude>META-INF/*.RSA</exclude>
|
||||||
|
</excludes>
|
||||||
|
</filter>
|
||||||
|
</filters>
|
||||||
|
<transformers>
|
||||||
|
<transformer implementation="org.apache.maven.plugins.shade.resource.ManifestResourceTransformer">
|
||||||
|
<mainClass>com.heibaiying.keyedstate.KeyedStateJob</mainClass>
|
||||||
|
</transformer>
|
||||||
|
</transformers>
|
||||||
|
</configuration>
|
||||||
|
</execution>
|
||||||
|
</executions>
|
||||||
|
</plugin>
|
||||||
|
</plugins>
|
||||||
|
|
||||||
|
<pluginManagement>
|
||||||
|
<plugins>
|
||||||
|
|
||||||
|
<!-- This improves the out-of-the-box experience in Eclipse by resolving some warnings. -->
|
||||||
|
<plugin>
|
||||||
|
<groupId>org.eclipse.m2e</groupId>
|
||||||
|
<artifactId>lifecycle-mapping</artifactId>
|
||||||
|
<version>1.0.0</version>
|
||||||
|
<configuration>
|
||||||
|
<lifecycleMappingMetadata>
|
||||||
|
<pluginExecutions>
|
||||||
|
<pluginExecution>
|
||||||
|
<pluginExecutionFilter>
|
||||||
|
<groupId>org.apache.maven.plugins</groupId>
|
||||||
|
<artifactId>maven-shade-plugin</artifactId>
|
||||||
|
<versionRange>[3.0.0,)</versionRange>
|
||||||
|
<goals>
|
||||||
|
<goal>shade</goal>
|
||||||
|
</goals>
|
||||||
|
</pluginExecutionFilter>
|
||||||
|
<action>
|
||||||
|
<ignore/>
|
||||||
|
</action>
|
||||||
|
</pluginExecution>
|
||||||
|
<pluginExecution>
|
||||||
|
<pluginExecutionFilter>
|
||||||
|
<groupId>org.apache.maven.plugins</groupId>
|
||||||
|
<artifactId>maven-compiler-plugin</artifactId>
|
||||||
|
<versionRange>[3.1,)</versionRange>
|
||||||
|
<goals>
|
||||||
|
<goal>testCompile</goal>
|
||||||
|
<goal>compile</goal>
|
||||||
|
</goals>
|
||||||
|
</pluginExecutionFilter>
|
||||||
|
<action>
|
||||||
|
<ignore/>
|
||||||
|
</action>
|
||||||
|
</pluginExecution>
|
||||||
|
</pluginExecutions>
|
||||||
|
</lifecycleMappingMetadata>
|
||||||
|
</configuration>
|
||||||
|
</plugin>
|
||||||
|
</plugins>
|
||||||
|
</pluginManagement>
|
||||||
|
</build>
|
||||||
|
|
||||||
|
<!-- This profile helps to make things run out of the box in IntelliJ -->
|
||||||
|
<!-- Its adds Flink's core classes to the runtime class path. -->
|
||||||
|
<!-- Otherwise they are missing in IntelliJ, because the dependency is 'provided' -->
|
||||||
|
<profiles>
|
||||||
|
<profile>
|
||||||
|
<id>add-dependencies-for-IDEA</id>
|
||||||
|
|
||||||
|
<activation>
|
||||||
|
<property>
|
||||||
|
<name>idea.version</name>
|
||||||
|
</property>
|
||||||
|
</activation>
|
||||||
|
|
||||||
|
<dependencies>
|
||||||
|
<dependency>
|
||||||
|
<groupId>org.apache.flink</groupId>
|
||||||
|
<artifactId>flink-java</artifactId>
|
||||||
|
<version>${flink.version}</version>
|
||||||
|
<scope>compile</scope>
|
||||||
|
</dependency>
|
||||||
|
<dependency>
|
||||||
|
<groupId>org.apache.flink</groupId>
|
||||||
|
<artifactId>flink-streaming-java_${scala.binary.version}</artifactId>
|
||||||
|
<version>${flink.version}</version>
|
||||||
|
<scope>compile</scope>
|
||||||
|
</dependency>
|
||||||
|
</dependencies>
|
||||||
|
</profile>
|
||||||
|
</profiles>
|
||||||
|
|
||||||
|
</project>
|
||||||
@ -0,0 +1,24 @@
|
|||||||
|
package com.heibaiying.keyedstate;
|
||||||
|
|
||||||
|
import org.apache.flink.api.java.tuple.Tuple2;
|
||||||
|
import org.apache.flink.streaming.api.datastream.DataStreamSource;
|
||||||
|
import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment;
|
||||||
|
|
||||||
|
|
||||||
|
public class KeyedStateJob {
|
||||||
|
|
||||||
|
public static void main(String[] args) throws Exception {
|
||||||
|
final StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
|
||||||
|
DataStreamSource<Tuple2<String, Long>> tuple2DataStreamSource = env.fromElements(
|
||||||
|
Tuple2.of("a", 50L), Tuple2.of("a", 80L), Tuple2.of("a", 400L),
|
||||||
|
Tuple2.of("a", 100L), Tuple2.of("a", 200L), Tuple2.of("a", 200L),
|
||||||
|
Tuple2.of("b", 100L), Tuple2.of("b", 200L), Tuple2.of("b", 200L),
|
||||||
|
Tuple2.of("b", 500L), Tuple2.of("b", 600L), Tuple2.of("b", 700L));
|
||||||
|
tuple2DataStreamSource
|
||||||
|
.keyBy(0)
|
||||||
|
.flatMap(new ThresholdWarning(100L, 3))
|
||||||
|
.printToErr();
|
||||||
|
env.execute("Managed Keyed State");
|
||||||
|
}
|
||||||
|
|
||||||
|
}
|
||||||
@ -0,0 +1,49 @@
|
|||||||
|
package com.heibaiying.keyedstate;
|
||||||
|
|
||||||
|
import org.apache.flink.api.common.functions.RichFlatMapFunction;
|
||||||
|
import org.apache.flink.api.common.state.ListState;
|
||||||
|
import org.apache.flink.api.common.state.ListStateDescriptor;
|
||||||
|
import org.apache.flink.api.java.tuple.Tuple2;
|
||||||
|
import org.apache.flink.configuration.Configuration;
|
||||||
|
import org.apache.flink.shaded.guava18.com.google.common.collect.Lists;
|
||||||
|
import org.apache.flink.util.Collector;
|
||||||
|
|
||||||
|
import java.util.ArrayList;
|
||||||
|
import java.util.List;
|
||||||
|
|
||||||
|
public class ThresholdWarning extends RichFlatMapFunction<Tuple2<String, Long>, Tuple2<String, List<Long>>> {
|
||||||
|
|
||||||
|
// 通过ListState来存储非正常数据的状态
|
||||||
|
private transient ListState<Long> abnormalData;
|
||||||
|
// 需要监控阈值
|
||||||
|
private Long threshold;
|
||||||
|
// 达到阈值多少次后触发报警
|
||||||
|
private Integer numberOfTimes;
|
||||||
|
|
||||||
|
ThresholdWarning(Long threshold, Integer numberOfTimes) {
|
||||||
|
this.threshold = threshold;
|
||||||
|
this.numberOfTimes = numberOfTimes;
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public void open(Configuration parameters) {
|
||||||
|
// 通过状态名称(句柄)获取状态实例，如果不存在则会自动创建
|
||||||
|
abnormalData = getRuntimeContext().getListState(new ListStateDescriptor<>("abnormalData", Long.class));
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public void flatMap(Tuple2<String, Long> value, Collector<Tuple2<String, List<Long>>> out) throws Exception {
|
||||||
|
Long inputValue = value.f1;
|
||||||
|
// 如果输入值超过阈值，则记录该次不正常的数据信息
|
||||||
|
if (inputValue >= threshold) {
|
||||||
|
abnormalData.add(inputValue);
|
||||||
|
}
|
||||||
|
ArrayList<Long> list = Lists.newArrayList(abnormalData.get().iterator());
|
||||||
|
// 如果不正常的数据出现达到一定次数，则输出报警信息
|
||||||
|
if (list.size() >= numberOfTimes) {
|
||||||
|
out.collect(Tuple2.of(value.f0 + " 超过指定阈值 ", list));
|
||||||
|
// 报警信息输出后，清空暂存的状态
|
||||||
|
abnormalData.clear();
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
@ -0,0 +1,54 @@
|
|||||||
|
package com.heibaiying.keyedstate;
|
||||||
|
|
||||||
|
import org.apache.flink.api.common.functions.RichFlatMapFunction;
|
||||||
|
import org.apache.flink.api.common.state.ListState;
|
||||||
|
import org.apache.flink.api.common.state.ListStateDescriptor;
|
||||||
|
import org.apache.flink.api.common.state.StateTtlConfig;
|
||||||
|
import org.apache.flink.api.common.time.Time;
|
||||||
|
import org.apache.flink.api.java.tuple.Tuple2;
|
||||||
|
import org.apache.flink.configuration.Configuration;
|
||||||
|
import org.apache.flink.shaded.guava18.com.google.common.collect.Lists;
|
||||||
|
import org.apache.flink.util.Collector;
|
||||||
|
|
||||||
|
import java.util.ArrayList;
|
||||||
|
import java.util.List;
|
||||||
|
|
||||||
|
public class ThresholdWarningWithTTL extends RichFlatMapFunction<Tuple2<String, Long>, Tuple2<String, List<Long>>> {
|
||||||
|
|
||||||
|
private transient ListState<Long> abnormalData;
|
||||||
|
private Long threshold;
|
||||||
|
private Integer numberOfTimes;
|
||||||
|
|
||||||
|
ThresholdWarningWithTTL(Long threshold, Integer numberOfTimes) {
|
||||||
|
this.threshold = threshold;
|
||||||
|
this.numberOfTimes = numberOfTimes;
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public void open(Configuration parameters) {
|
||||||
|
StateTtlConfig ttlConfig = StateTtlConfig
|
||||||
|
// 设置有效期为 10 秒
|
||||||
|
.newBuilder(Time.seconds(10))
|
||||||
|
// 设置有效期更新规则，这里设置为当创建和写入时，都重置其有效期到规定的10秒
|
||||||
|
.setUpdateType(StateTtlConfig.UpdateType.OnCreateAndWrite)
|
||||||
|
// 设置只要值过期就不可见，另外一个可选值是 ReturnExpiredIfNotCleanedUp，代表即使值过期了，但如果还没有被删除，就是可见的
|
||||||
|
.setStateVisibility(StateTtlConfig.StateVisibility.NeverReturnExpired)
|
||||||
|
.build();
|
||||||
|
ListStateDescriptor<Long> descriptor = new ListStateDescriptor<>("abnormalData", Long.class);
|
||||||
|
descriptor.enableTimeToLive(ttlConfig);
|
||||||
|
this.abnormalData = getRuntimeContext().getListState(descriptor);
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public void flatMap(Tuple2<String, Long> value, Collector<Tuple2<String, List<Long>>> out) throws Exception {
|
||||||
|
Long inputValue = value.f1;
|
||||||
|
if (inputValue >= threshold) {
|
||||||
|
abnormalData.add(inputValue);
|
||||||
|
}
|
||||||
|
ArrayList<Long> list = Lists.newArrayList(abnormalData.get().iterator());
|
||||||
|
if (list.size() >= numberOfTimes) {
|
||||||
|
out.collect(Tuple2.of(value.f0 + " 超过指定阈值 ", list));
|
||||||
|
abnormalData.clear();
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
@ -0,0 +1,26 @@
|
|||||||
|
package com.heibaiying.operatorstate;
|
||||||
|
|
||||||
|
import org.apache.flink.api.java.tuple.Tuple2;
|
||||||
|
import org.apache.flink.streaming.api.datastream.DataStreamSource;
|
||||||
|
import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment;
|
||||||
|
|
||||||
|
|
||||||
|
public class OperatorStateJob {
|
||||||
|
|
||||||
|
public static void main(String[] args) throws Exception {
|
||||||
|
final StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
|
||||||
|
// 开启检查点机制
|
||||||
|
env.enableCheckpointing(1000);
|
||||||
|
// 设置并行度为1
|
||||||
|
DataStreamSource<Tuple2<String, Long>> tuple2DataStreamSource = env.setParallelism(1).fromElements(
|
||||||
|
Tuple2.of("a", 50L), Tuple2.of("a", 80L), Tuple2.of("a", 400L),
|
||||||
|
Tuple2.of("a", 100L), Tuple2.of("a", 200L), Tuple2.of("a", 200L),
|
||||||
|
Tuple2.of("b", 100L), Tuple2.of("b", 200L), Tuple2.of("b", 200L),
|
||||||
|
Tuple2.of("b", 500L), Tuple2.of("b", 600L), Tuple2.of("b", 700L));
|
||||||
|
tuple2DataStreamSource
|
||||||
|
.flatMap(new ThresholdWarning(100L, 3))
|
||||||
|
.printToErr();
|
||||||
|
env.execute("Managed Keyed State");
|
||||||
|
}
|
||||||
|
|
||||||
|
}
|
||||||
@ -0,0 +1,72 @@
|
|||||||
|
package com.heibaiying.operatorstate;
|
||||||
|
|
||||||
|
import org.apache.flink.api.common.functions.RichFlatMapFunction;
|
||||||
|
import org.apache.flink.api.common.state.ListState;
|
||||||
|
import org.apache.flink.api.common.state.ListStateDescriptor;
|
||||||
|
import org.apache.flink.api.common.typeinfo.TypeHint;
|
||||||
|
import org.apache.flink.api.common.typeinfo.TypeInformation;
|
||||||
|
import org.apache.flink.api.java.tuple.Tuple2;
|
||||||
|
import org.apache.flink.runtime.state.FunctionInitializationContext;
|
||||||
|
import org.apache.flink.runtime.state.FunctionSnapshotContext;
|
||||||
|
import org.apache.flink.streaming.api.checkpoint.CheckpointedFunction;
|
||||||
|
import org.apache.flink.util.Collector;
|
||||||
|
|
||||||
|
import java.util.ArrayList;
|
||||||
|
import java.util.List;
|
||||||
|
|
||||||
|
public class ThresholdWarning extends RichFlatMapFunction<Tuple2<String, Long>, Tuple2<String, List<Tuple2<String, Long>>>> implements CheckpointedFunction {
|
||||||
|
|
||||||
|
// 非正常数据
|
||||||
|
private List<Tuple2<String, Long>> bufferedData;
|
||||||
|
// checkPointedState
|
||||||
|
private transient ListState<Tuple2<String, Long>> checkPointedState;
|
||||||
|
// 需要监控的阈值
|
||||||
|
private Long threshold;
|
||||||
|
// 次数
|
||||||
|
private Integer numberOfTimes;
|
||||||
|
|
||||||
|
ThresholdWarning(Long threshold, Integer numberOfTimes) {
|
||||||
|
this.threshold = threshold;
|
||||||
|
this.numberOfTimes = numberOfTimes;
|
||||||
|
this.bufferedData = new ArrayList<>();
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public void initializeState(FunctionInitializationContext context) throws Exception {
|
||||||
|
// 注意这里获取的是OperatorStateStore
|
||||||
|
checkPointedState = context.getOperatorStateStore().getListState(new ListStateDescriptor<>("abnormalData",
|
||||||
|
TypeInformation.of(new TypeHint<Tuple2<String, Long>>() {
|
||||||
|
})));
|
||||||
|
// 如果发生重启，则需要从快照中将状态进行恢复
|
||||||
|
if (context.isRestored()) {
|
||||||
|
for (Tuple2<String, Long> element : checkPointedState.get()) {
|
||||||
|
bufferedData.add(element);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public void flatMap(Tuple2<String, Long> value, Collector<Tuple2<String, List<Tuple2<String, Long>>>> out) {
|
||||||
|
Long inputValue = value.f1;
|
||||||
|
// 超过阈值则进行记录
|
||||||
|
if (inputValue >= threshold) {
|
||||||
|
bufferedData.add(value);
|
||||||
|
}
|
||||||
|
// 超过指定次数则输出报警信息
|
||||||
|
if (bufferedData.size() >= numberOfTimes) {
|
||||||
|
// 顺便输出状态实例的hashcode
|
||||||
|
out.collect(Tuple2.of(checkPointedState.hashCode() + "阈值警报！", bufferedData));
|
||||||
|
bufferedData.clear();
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public void snapshotState(FunctionSnapshotContext context) throws Exception {
|
||||||
|
// 在进行快照时，将数据存储到checkPointedState
|
||||||
|
checkPointedState.clear();
|
||||||
|
for (Tuple2<String, Long> element : bufferedData) {
|
||||||
|
checkPointedState.add(element);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
}
|
||||||
@ -0,0 +1,23 @@
|
|||||||
|
################################################################################
|
||||||
|
# Licensed to the Apache Software Foundation (ASF) under one
|
||||||
|
# or more contributor license agreements. See the NOTICE file
|
||||||
|
# distributed with this work for additional information
|
||||||
|
# regarding copyright ownership. The ASF licenses this file
|
||||||
|
# to you under the Apache License, Version 2.0 (the
|
||||||
|
# "License"); you may not use this file except in compliance
|
||||||
|
# with the License. You may obtain a copy of the License at
|
||||||
|
#
|
||||||
|
# http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
#
|
||||||
|
# Unless required by applicable law or agreed to in writing, software
|
||||||
|
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||||
|
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||||
|
# See the License for the specific language governing permissions and
|
||||||
|
# limitations under the License.
|
||||||
|
################################################################################
|
||||||
|
|
||||||
|
log4j.rootLogger=INFO, console
|
||||||
|
|
||||||
|
log4j.appender.console=org.apache.log4j.ConsoleAppender
|
||||||
|
log4j.appender.console.layout=org.apache.log4j.PatternLayout
|
||||||
|
log4j.appender.console.layout.ConversionPattern=%d{HH:mm:ss,SSS} %-5p %-60c %x - %m%n
|
||||||
@ -220,7 +220,6 @@ ReScale 这个单词具有重新缩放的意义，其对应的操作也是如此
|
|||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
### 3.4 Broadcasting [DataStream → DataStream]
|
### 3.4 Broadcasting [DataStream → DataStream]
|
||||||
|
|
||||||
将数据分发到所有分区上。通常用于小数据集与大数据集进行关联的情况下，此时可以将小数据集广播到所有分区上，避免频繁的跨分区关联，通过 broadcast 方法进行实现：
|
将数据分发到所有分区上。通常用于小数据集与大数据集进行关联的情况下，此时可以将小数据集广播到所有分区上，避免频繁的跨分区关联，通过 broadcast 方法进行实现：
|
||||||
@ -282,7 +281,7 @@ someStream.map(...).disableChaining();
|
|||||||
|
|
||||||
### 4.3 slotSharingGroup
|
### 4.3 slotSharingGroup
|
||||||
|
|
||||||
slot 是任务管理器 (TaskManager) 所拥有资源的固定子集，每个操作 (operation) 的子任务 (sub task) 都需要获取 slot 来执行计算，但有的操作是资源密集型的，有的则是 CPU 密集型的，为了更好地利用资源，Flink 允许不同操作的子任务被部署到同一 slot 中。slotSharingGroup 用于设置操作的 slot 共享组 (slot sharing group) ，Flink 会将具有相同 slot 共享组的操作放到同一个 slot 中 。示例如下：
|
slot 是任务管理器 (TaskManager) 所拥有资源的固定子集，每个操作 (operation) 的子任务 (sub task) 都需要获取 slot 来执行计算，但每个操作所需要资源的大小都是不相同的，为了更好地利用资源，Flink 允许不同操作的子任务被部署到同一 slot 中。slotSharingGroup 用于设置操作的 slot 共享组 (slot sharing group) ，Flink 会将具有相同 slot 共享组的操作放到同一个 slot 中 。示例如下：
|
||||||
|
|
||||||
```java
|
```java
|
||||||
someStream.filter(...).slotSharingGroup("slotSharingGroupName");
|
someStream.filter(...).slotSharingGroup("slotSharingGroupName");
|
||||||
|
|||||||
138
notes/Flink核心概念综述.md
Normal file
138
notes/Flink核心概念综述.md
Normal file
@ -0,0 +1,138 @@
|
|||||||
|
# Flink 核心概念综述
|
||||||
|
|
||||||
|
## 一、Flink 简介
|
||||||
|
|
||||||
|
Apache Flink 诞生于柏林工业大学的一个研究性项目，原名 StratoSphere 。2014 年，由 StratoSphere 项目孵化出 Flink，并于同年捐赠 Apache，之后成为 Apache 的顶级项目。2019 年 1 年，阿里巴巴收购了 Flink 的母公司 Data Artisans，并宣布开源内部的 Blink，Blink 是阿里巴巴基于 Flink 优化后的版本，增加了大量的新功能，并在性能和稳定性上进行了各种优化，经历过阿里内部多种复杂业务的挑战和检验。同时阿里巴巴也表示会逐步将这些新功能和特性 Merge 回社区版本的 Flink 中，因此 Flink 成为目前最为火热的大数据处理框架。
|
||||||
|
|
||||||
|
简单来说，Flink 是一个分布式的流处理框架，它能够对有界和无界的数据流进行高效的处理。Flink 的核心是流处理，当然它也能支持批处理，Flink 将批处理看成是流处理的一种特殊情况，即数据流是有明确界限的。这和 Spark Streaming 的思想是完全相反的，Spark Streaming 的核心是批处理，它将流处理看成是批处理的一种特殊情况， 即把数据流进行极小粒度的拆分，拆分为多个微批处理。
|
||||||
|
|
||||||
|
Flink 有界数据流和无界数据流：
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
Spark Streaming 数据流的拆分：
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
## 二、Flink 核心架构
|
||||||
|
|
||||||
|
Flink 采用分层的架构设计，从而保证各层在功能和职责上的清晰。如下图所示，由上而下分别是 API & Libraries 层、Runtime 核心层以及物理部署层：
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
### 2.1 API & Libraries 层
|
||||||
|
|
||||||
|
这一层主要提供了编程 API 和 顶层类库：
|
||||||
|
|
||||||
|
+ 编程 API : 用于进行流处理的 DataStream API 和用于进行批处理的 DataSet API；
|
||||||
|
+ 顶层类库：包括用于复杂事件处理的 CEP 库；用于结构化数据查询的 SQL & Table 库，以及基于批处理的机器学习库 FlinkML 和 图形处理库 Gelly。
|
||||||
|
|
||||||
|
### 2.2 Runtime 核心层
|
||||||
|
|
||||||
|
这一层是 Flink 分布式计算框架的核心实现层，包括作业转换，任务调度，资源分配，任务执行等功能，基于这一层的实现，可以在流式引擎下同时运行流处理程序和批处理程序。
|
||||||
|
|
||||||
|
### 2.3 物理部署层
|
||||||
|
|
||||||
|
Flink 的物理部署层，用于支持在不同平台上部署运行 Flink 应用。
|
||||||
|
|
||||||
|
## 三、Flink 分层 API
|
||||||
|
|
||||||
|
在上面介绍的 API & Libraries 这一层，Flink 又进行了更为具体的划分。具体如下：
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
按照如上的层次结构，API 的一致性由下至上依次递增，接口的表现能力由下至上依次递减，各层的核心功能如下：
|
||||||
|
|
||||||
|
### 3.1 SQL & Table API
|
||||||
|
|
||||||
|
SQL & Table API 同时适用于批处理和流处理，这意味着你可以对有界数据流和无界数据流以相同的语义进行查询，并产生相同的结果。除了基本查询外， 它还支持自定义的标量函数，聚合函数以及表值函数，可以满足多样化的查询需求。
|
||||||
|
|
||||||
|
### 3.2 DataStream & DataSet API
|
||||||
|
|
||||||
|
DataStream & DataSet API 是 Flink 数据处理的核心 API，支持使用 Java 语言或 Scala 语言进行调用，提供了数据读取，数据转换和数据输出等一系列常用操作的封装。
|
||||||
|
|
||||||
|
### 3.3 Stateful Stream Processing
|
||||||
|
|
||||||
|
Stateful Stream Processing 是最低级别的抽象，它通过 Process Function 函数内嵌到 DataStream API 中。 Process Function 是 Flink 提供的最底层 API，具有最大的灵活性，允许开发者对于时间和状态进行细粒度的控制。
|
||||||
|
|
||||||
|
## 四、Flink 集群架构
|
||||||
|
|
||||||
|
### 4.1 核心组件
|
||||||
|
|
||||||
|
按照上面的介绍，Flink 核心架构的第二层是 Runtime 层， 该层采用标准的 Master - Slave 结构， 其中，Master 部分又包含了三个核心组件：Dispatcher、ResourceManager 和 JobManager，而 Slave 则主要是 TaskManager 进程。它们的功能分别如下：
|
||||||
|
|
||||||
|
- **JobManagers** (也称为 *masters*) ： 整个作业 (Job) 的主要管理者，负责调度任务 (tasks)、协调检查点 (checkpoints) 、协调故障恢复等。每个 Job 至少有一个 JobManager；高可用部署下可以有多个 JobManagers，其中一个作为 *leader*，其余的则处于 *standby* 状态。
|
||||||
|
- **TaskManagers** (也称为 *workers*) ：负责子任务 (subtasks) 的执行。每个 Job 至少会有一个 TaskManager。
|
||||||
|
- **Dispatcher**：负责接收客户端的作业，并启动 JobManager。
|
||||||
|
- **ResourceManager** ：负责集群资源的协调和管理。
|
||||||
|
|
||||||
|
以 Standalone 模式为例，它们之间的运行流程如下：
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
+ 用户通过 Client 将作业 ( Job) 提交给 Master 时，此时需要先经过 Dispatcher；
|
||||||
|
+ 当 Dispatcher 收到客户端的请求之后，会生成一个 JobManager，接着 JobManager 进程向 ResourceManager 申请资源来启动 TaskManager；
|
||||||
|
+ TaskManager 启动之后，它需要将自己注册到 ResourceManager 上。注册完成后， JobManager 再将具体的 Task 任务分发给这个 TaskManager 去执行。
|
||||||
|
|
||||||
|
### 4.2 Task & SubTask
|
||||||
|
|
||||||
|
在上面我们提到 JobManagers 负责管理的是 Task ，而 TaskManagers 负责执行的则是 SubTask，这里解释一下任务 Task 和子任务 SubTask 两者间的关系：
|
||||||
|
|
||||||
|
在执行分布式计算时，Flink 将可以链接的操作 (operators) 链接到一起，这就是 Task。之所以这样做， 是为了减少线程间切换和缓冲而导致的开销，在降低延迟的同时可以提高整体的吞吐量。 但不是所有的 operator 都可以被链接，如下 keyBy 等操作会导致网络 shuffle 和重分区，因此其就不能被链接，只能被单独作为一个 Task。 简单来说，一个 Task 就是一个可以链接的最小的操作链 (Operator Chains) 。如下图，source 和 map 算子被链接到一块，因此整个作业就只有三个 Task：
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
解释完 Task ，我们在解释一下什么是 SubTask，其准确的翻译是： *A subtask is one parallel slice of a task*，即一个 Task 可以按照其并行度拆分为多个 SubTask。如上图，source & map 具有两个并行度，KeyBy 具有两个并行度，Sink 具有一个并行度，因此整个虽然只有 3 个 Task，但是却有 5 个 SubTask，每个 SubTask 都是一个单独的线程。
|
||||||
|
|
||||||
|
Jobmanager 负责定义和拆分这些 SubTask，并将其交给 Taskmanagers 来执行。想要成功执行这些任务，Taskmanagers 还必须要具备运行这些 SubTask 所需要的计算资源和内存资源。
|
||||||
|
|
||||||
|
### 4.3 资源管理
|
||||||
|
|
||||||
|
ResourceManager 对资源的管理是以 Slot 为单位的，Slot 是一组固定大小的资源的合集。 在上面的过程中，JobManager 进程向 ResourceManager 申请资源来启动 TaskManager，申请的就是 Slot 资源，此时上面 5 个 SubTasks 与 Slots 的对应情况则可能如下：
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
这时每个 SubTask 线程运行在一个独立的 TaskSlot， 它们共享所属的 TaskManager 进程的TCP 连接（通过多路复用技术）和心跳信息 (heartbeat messages)，从而可以降低整体的性能开销。此时看似是最好的情况，但是每个操作需要的资源都是不尽相同的，这里假设该作业 keyBy 操作所需资源的数量比 Sink 多很多 ，那么此时 Sink 所在 Slot 的资源就没有得到有效的利用。
|
||||||
|
|
||||||
|
基于这个原因，Flink 允许多个 subtasks 共享 slots，即使它们是不同 tasks 的 subtasks，但只要它们来自同一个 Job 就可以。假设上面 souce & map 和 keyBy 的并行度调整为 6，而 Slot 的数量不变，此时情况如下：
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
可以看到一个 Task Slot 中运行了多个 SubTask 子任务，此时每个子任务仍然在一个独立的线程中执行，只不过共享一组 Sot 资源而已。那么 Flink 到底如何确定一个 Job 至少需要多少个 Slot 呢？Flink 对于这个问题的处理很简单，默认情况一个 Job 所需要的 Slot 的数量就等于其 Operation 操作的最高并行度。如下， A，B，D 操作的并行度为 4，而 C，E 操作的并行度为 2，那么此时整个 Job 就需要至少四个 Slots 来完成。通过这个机制，Flink 就可以不必去关心一个 Job 到底会被拆分为多少个 Tasks 和 SubTasks。
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
### 4.4 组件通讯
|
||||||
|
|
||||||
|
Flink 的所有组件都基于 Actor System 来进行通讯。Actor system是多种角色的 actor 的容器，它提供调度，配置，日志记录等多种服务，并包含一个可以启动所有 actor 的线程池，如果 actor 是本地的，则消息通过共享内存进行共享，但如果 actor 是远程的，则通过 RPC 的调用来传递消息。
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
## 五、Flink 的优点
|
||||||
|
|
||||||
|
最后来介绍一下 Flink 的优点：
|
||||||
|
|
||||||
|
+ Flink 是基于事件驱动 (Event-driven) 的应用，能够同时支持流处理和批处理；
|
||||||
|
+ 基于内存的计算，能够保证高吞吐和低延迟，具有优越的性能表现；
|
||||||
|
+ 支持精确一次 (Exactly-once) 语意，能够完美地保证一致性和正确性；
|
||||||
|
+ 分层 API ，能够满足各个层次的开发需求；
|
||||||
|
+ 支持高可用配置，支持保存点机制，能够提供安全性和稳定性上的保证；
|
||||||
|
+ 多样化的部署方式，支持本地，远端，云端等多种部署方案；
|
||||||
|
+ 具有横向扩展架构，能够按照用户的需求进行动态扩容；
|
||||||
|
+ 活跃度极高的社区和完善的生态圈的支持。
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
## 参考资料
|
||||||
|
|
||||||
|
+ [Dataflow Programming Model](https://ci.apache.org/projects/flink/flink-docs-release-1.9/concepts/programming-model.html)
|
||||||
|
+ [Distributed Runtime Environment](https://ci.apache.org/projects/flink/flink-docs-release-1.9/concepts/runtime.html)
|
||||||
|
+ [Component Stack](https://ci.apache.org/projects/flink/flink-docs-release-1.9/internals/components.html)
|
||||||
|
+ [Flink on Yarn/K8s 原理剖析及实践]( https://www.infoq.cn/article/UFeFrdqSlqI3HyRHbPNm )
|
||||||
|
+ Fabian Hueske , Vasiliki Kalavri . 《Stream Processing with Apache Flink》. O'Reilly Media . 2019-4-30
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
337
notes/Flink状态管理.md
Normal file
337
notes/Flink状态管理.md
Normal file
@ -0,0 +1,337 @@
|
|||||||
|
# Flink 状态管理
|
||||||
|
|
||||||
|
## 一、状态分类
|
||||||
|
|
||||||
|
相对于其他流计算框架，Flink 一个比较重要的特性就是其支持有状态计算。即你可以将中间的计算结果进行保存，并提供给后续的计算使用：
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
具体而言，Flink 又将状态 (State) 分为 Keyed State 与 Operator State：
|
||||||
|
|
||||||
|
### 2.1 算子状态
|
||||||
|
|
||||||
|
算子状态 (Operator State)：顾名思义，状态是和算子进行绑定的，一个算子的状态不能被其他算子所访问到。官方文档上对 Operator State 的解释是：*each operator state is bound to one parallel operator instance*，所以更为确切的说一个算子状态是与一个并发的算子实例所绑定的，即假设算子的并行度是 2，那么其应有两个对应的算子状态：
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
### 2.2 键控状态
|
||||||
|
|
||||||
|
键控状态 (Keyed State) ：是一种特殊的算子状态，即状态是根据 key 值进行区分的，Flink 会为每类键值维护一个状态实例。如下图所示，每个颜色代表不同 key 值，对应四个不同的状态实例。需要注意的是键控状态只能在 `KeyedStream` 上进行使用，我们可以通过 `stream.keyBy(...)` 来得到 `KeyedStream` 。
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
## 二、状态编程
|
||||||
|
|
||||||
|
### 2.1 键控状态
|
||||||
|
|
||||||
|
Flink 提供了以下数据格式来管理和存储键控状态 (Keyed State)：
|
||||||
|
|
||||||
|
- **ValueState**：存储单值类型的状态。可以使用 `update(T)` 进行更新，并通过 `T value()` 进行检索。
|
||||||
|
- **ListState**：存储列表类型的状态。可以使用 `add(T)` 或 `addAll(List)` 添加元素；并通过 `get()` 获得整个列表。
|
||||||
|
- **ReducingState**：用于存储经过 ReduceFunction 计算后的结果，使用 `add(T)` 增加元素。
|
||||||
|
- **AggregatingState**：用于存储经过 AggregatingState 计算后的结果，使用 `add(IN)` 添加元素。
|
||||||
|
- **FoldingState**：已被标识为废弃，会在未来版本中移除，官方推荐使用 `AggregatingState` 代替。
|
||||||
|
- **MapState**：维护 Map 类型的状态。
|
||||||
|
|
||||||
|
以上所有增删改查方法不必硬记，在使用时通过语法提示来调用即可。这里给出一个具体的使用示例：假设我们正在开发一个监控系统，当监控数据超过阈值一定次数后，需要发出报警信息。这里之所以要达到一定次数，是因为由于偶发原因，偶尔一次超过阈值并不能代表什么，故需要达到一定次数后才触发报警，这就需要使用到 Flink 的状态编程。相关代码如下：
|
||||||
|
|
||||||
|
```java
|
||||||
|
public class ThresholdWarning extends
|
||||||
|
RichFlatMapFunction<Tuple2<String, Long>, Tuple2<String, List<Long>>> {
|
||||||
|
|
||||||
|
// 通过ListState来存储非正常数据的状态
|
||||||
|
private transient ListState<Long> abnormalData;
|
||||||
|
// 需要监控的阈值
|
||||||
|
private Long threshold;
|
||||||
|
// 触发报警的次数
|
||||||
|
private Integer numberOfTimes;
|
||||||
|
|
||||||
|
ThresholdWarning(Long threshold, Integer numberOfTimes) {
|
||||||
|
this.threshold = threshold;
|
||||||
|
this.numberOfTimes = numberOfTimes;
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public void open(Configuration parameters) {
|
||||||
|
// 通过状态名称(句柄)获取状态实例，如果不存在则会自动创建
|
||||||
|
abnormalData = getRuntimeContext().getListState(
|
||||||
|
new ListStateDescriptor<>("abnormalData", Long.class));
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public void flatMap(Tuple2<String, Long> value, Collector<Tuple2<String, List<Long>>> out)
|
||||||
|
throws Exception {
|
||||||
|
Long inputValue = value.f1;
|
||||||
|
// 如果输入值超过阈值，则记录该次不正常的数据信息
|
||||||
|
if (inputValue >= threshold) {
|
||||||
|
abnormalData.add(inputValue);
|
||||||
|
}
|
||||||
|
ArrayList<Long> list = Lists.newArrayList(abnormalData.get().iterator());
|
||||||
|
// 如果不正常的数据出现达到一定次数，则输出报警信息
|
||||||
|
if (list.size() >= numberOfTimes) {
|
||||||
|
out.collect(Tuple2.of(value.f0 + " 超过指定阈值 ", list));
|
||||||
|
// 报警信息输出后，清空状态
|
||||||
|
abnormalData.clear();
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
```
|
||||||
|
|
||||||
|
调用自定义的状态监控，这里我们使用 a，b 来代表不同类型的监控数据，分别对其数据进行监控：
|
||||||
|
|
||||||
|
```java
|
||||||
|
final StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
|
||||||
|
DataStreamSource<Tuple2<String, Long>> tuple2DataStreamSource = env.fromElements(
|
||||||
|
Tuple2.of("a", 50L), Tuple2.of("a", 80L), Tuple2.of("a", 400L),
|
||||||
|
Tuple2.of("a", 100L), Tuple2.of("a", 200L), Tuple2.of("a", 200L),
|
||||||
|
Tuple2.of("b", 100L), Tuple2.of("b", 200L), Tuple2.of("b", 200L),
|
||||||
|
Tuple2.of("b", 500L), Tuple2.of("b", 600L), Tuple2.of("b", 700L));
|
||||||
|
tuple2DataStreamSource
|
||||||
|
.keyBy(0)
|
||||||
|
.flatMap(new ThresholdWarning(100L, 3)) // 超过100的阈值3次后就进行报警
|
||||||
|
.printToErr();
|
||||||
|
env.execute("Managed Keyed State");
|
||||||
|
```
|
||||||
|
|
||||||
|
输出如下结果如下：
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
### 2.2 状态有效期
|
||||||
|
|
||||||
|
以上任何类型的 keyed state 都支持配置有效期 (TTL) ，示例如下：
|
||||||
|
|
||||||
|
```java
|
||||||
|
StateTtlConfig ttlConfig = StateTtlConfig
|
||||||
|
// 设置有效期为 10 秒
|
||||||
|
.newBuilder(Time.seconds(10))
|
||||||
|
// 设置有效期更新规则，这里设置为当创建和写入时，都重置其有效期到规定的10秒
|
||||||
|
.setUpdateType(StateTtlConfig.UpdateType.OnCreateAndWrite)
|
||||||
|
/*设置只要值过期就不可见，另外一个可选值是ReturnExpiredIfNotCleanedUp，
|
||||||
|
代表即使值过期了，但如果还没有被物理删除，就是可见的*/
|
||||||
|
.setStateVisibility(StateTtlConfig.StateVisibility.NeverReturnExpired)
|
||||||
|
.build();
|
||||||
|
ListStateDescriptor<Long> descriptor = new ListStateDescriptor<>("abnormalData", Long.class);
|
||||||
|
descriptor.enableTimeToLive(ttlConfig);
|
||||||
|
```
|
||||||
|
|
||||||
|
### 2.3 算子状态
|
||||||
|
|
||||||
|
相比于键控状态，算子状态目前支持的存储类型只有以下三种：
|
||||||
|
|
||||||
|
- **ListState**：存储列表类型的状态。
|
||||||
|
- **UnionListState**：存储列表类型的状态，与 ListState 的区别在于：如果并行度发生变化，ListState 会将该算子的所有并发的状态实例进行汇总，然后均分给新的 Task；而 UnionListState 只是将所有并发的状态实例汇总起来，具体的划分行为则由用户进行定义。
|
||||||
|
- **BroadcastState**：用于广播算子状态。
|
||||||
|
|
||||||
|
这里我们继续沿用上面的例子，假设此时我们不需要区分监控数据的类型，只要有监控数据超过阈值并达到指定的次数后，就进行报警，代码如下：
|
||||||
|
|
||||||
|
```java
|
||||||
|
public class ThresholdWarning extends RichFlatMapFunction<Tuple2<String, Long>,
|
||||||
|
Tuple2<String, List<Tuple2<String, Long>>>> implements CheckpointedFunction {
|
||||||
|
|
||||||
|
// 非正常数据
|
||||||
|
private List<Tuple2<String, Long>> bufferedData;
|
||||||
|
// checkPointedState
|
||||||
|
private transient ListState<Tuple2<String, Long>> checkPointedState;
|
||||||
|
// 需要监控的阈值
|
||||||
|
private Long threshold;
|
||||||
|
// 次数
|
||||||
|
private Integer numberOfTimes;
|
||||||
|
|
||||||
|
ThresholdWarning(Long threshold, Integer numberOfTimes) {
|
||||||
|
this.threshold = threshold;
|
||||||
|
this.numberOfTimes = numberOfTimes;
|
||||||
|
this.bufferedData = new ArrayList<>();
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public void initializeState(FunctionInitializationContext context) throws Exception {
|
||||||
|
// 注意这里获取的是OperatorStateStore
|
||||||
|
checkPointedState = context.getOperatorStateStore().
|
||||||
|
getListState(new ListStateDescriptor<>("abnormalData",
|
||||||
|
TypeInformation.of(new TypeHint<Tuple2<String, Long>>() {
|
||||||
|
})));
|
||||||
|
// 如果发生重启，则需要从快照中将状态进行恢复
|
||||||
|
if (context.isRestored()) {
|
||||||
|
for (Tuple2<String, Long> element : checkPointedState.get()) {
|
||||||
|
bufferedData.add(element);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public void flatMap(Tuple2<String, Long> value,
|
||||||
|
Collector<Tuple2<String, List<Tuple2<String, Long>>>> out) {
|
||||||
|
Long inputValue = value.f1;
|
||||||
|
// 超过阈值则进行记录
|
||||||
|
if (inputValue >= threshold) {
|
||||||
|
bufferedData.add(value);
|
||||||
|
}
|
||||||
|
// 超过指定次数则输出报警信息
|
||||||
|
if (bufferedData.size() >= numberOfTimes) {
|
||||||
|
// 顺便输出状态实例的hashcode
|
||||||
|
out.collect(Tuple2.of(checkPointedState.hashCode() + "阈值警报！", bufferedData));
|
||||||
|
bufferedData.clear();
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public void snapshotState(FunctionSnapshotContext context) throws Exception {
|
||||||
|
// 在进行快照时，将数据存储到checkPointedState
|
||||||
|
checkPointedState.clear();
|
||||||
|
for (Tuple2<String, Long> element : bufferedData) {
|
||||||
|
checkPointedState.add(element);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
```
|
||||||
|
|
||||||
|
调用自定义算子状态，这里需要将并行度设置为 1：
|
||||||
|
|
||||||
|
```java
|
||||||
|
final StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
|
||||||
|
// 开启检查点机制
|
||||||
|
env.enableCheckpointing(1000);
|
||||||
|
// 设置并行度为1
|
||||||
|
DataStreamSource<Tuple2<String, Long>> tuple2DataStreamSource = env.setParallelism(1).fromElements(
|
||||||
|
Tuple2.of("a", 50L), Tuple2.of("a", 80L), Tuple2.of("a", 400L),
|
||||||
|
Tuple2.of("a", 100L), Tuple2.of("a", 200L), Tuple2.of("a", 200L),
|
||||||
|
Tuple2.of("b", 100L), Tuple2.of("b", 200L), Tuple2.of("b", 200L),
|
||||||
|
Tuple2.of("b", 500L), Tuple2.of("b", 600L), Tuple2.of("b", 700L));
|
||||||
|
tuple2DataStreamSource
|
||||||
|
.flatMap(new ThresholdWarning(100L, 3))
|
||||||
|
.printToErr();
|
||||||
|
env.execute("Managed Keyed State");
|
||||||
|
}
|
||||||
|
```
|
||||||
|
|
||||||
|
此时输出如下：
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
在上面的调用代码中，我们将程序的并行度设置为 1，可以看到三次输出中状态实例的 hashcode 全是一致的，证明它们都同一个状态实例。假设将并行度设置为 2，此时输出如下：
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
可以看到此时两次输出中状态实例的 hashcode 是不一致的，代表它们不是同一个状态实例，这也就是上文提到的，一个算子状态是与一个并发的算子实例所绑定的。同时这里只输出两次，是因为在并发处理的情况下，线程 1 可能拿到 5 个非正常值，线程 2 可能拿到 4 个非正常值，因为要大于 3 次才能输出，所以在这种情况下就会出现只输出两条记录的情况，所以需要将程序的并行度设置为 1。
|
||||||
|
|
||||||
|
## 三、检查点机制
|
||||||
|
|
||||||
|
### 3.1 CheckPoints
|
||||||
|
|
||||||
|
为了使 Flink 的状态具有良好的容错性，Flink 提供了检查点机制 (CheckPoints) 。通过检查点机制，Flink 定期在数据流上生成 checkpoint barrier ，当某个算子收到 barrier 时，即会基于当前状态生成一份快照，然后再将该 barrier 传递到下游算子，下游算子接收到该 barrier 后，也基于当前状态生成一份快照，依次传递直至到最后的 Sink 算子上。当出现异常后，Flink 就可以根据最近的一次的快照数据将所有算子恢复到先前的状态。
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
### 3.2 开启检查点
|
||||||
|
|
||||||
|
默认情况下，检查点机制是关闭的，需要在程序中进行开启：
|
||||||
|
|
||||||
|
```java
|
||||||
|
// 开启检查点机制，并指定状态检查点之间的时间间隔
|
||||||
|
env.enableCheckpointing(1000);
|
||||||
|
|
||||||
|
// 其他可选配置如下：
|
||||||
|
// 设置语义
|
||||||
|
env.getCheckpointConfig().setCheckpointingMode(CheckpointingMode.EXACTLY_ONCE);
|
||||||
|
// 设置两个检查点之间的最小时间间隔
|
||||||
|
env.getCheckpointConfig().setMinPauseBetweenCheckpoints(500);
|
||||||
|
// 设置执行Checkpoint操作时的超时时间
|
||||||
|
env.getCheckpointConfig().setCheckpointTimeout(60000);
|
||||||
|
// 设置最大并发执行的检查点的数量
|
||||||
|
env.getCheckpointConfig().setMaxConcurrentCheckpoints(1);
|
||||||
|
// 将检查点持久化到外部存储
|
||||||
|
env.getCheckpointConfig().enableExternalizedCheckpoints(
|
||||||
|
ExternalizedCheckpointCleanup.RETAIN_ON_CANCELLATION);
|
||||||
|
// 如果有更近的保存点时，是否将作业回退到该检查点
|
||||||
|
env.getCheckpointConfig().setPreferCheckpointForRecovery(true);
|
||||||
|
```
|
||||||
|
|
||||||
|
### 3.3 保存点机制
|
||||||
|
|
||||||
|
保存点机制 (Savepoints) 是检查点机制的一种特殊的实现，它允许你通过手工的方式来触发 Checkpoint，并将结果持久化存储到指定路径中，主要用于避免 Flink 集群在重启或升级时导致状态丢失。示例如下：
|
||||||
|
|
||||||
|
```shell
|
||||||
|
# 触发指定id的作业的Savepoint，并将结果存储到指定目录下
|
||||||
|
bin/flink savepoint :jobId [:targetDirectory]
|
||||||
|
```
|
||||||
|
|
||||||
|
更多命令和配置可以参考官方文档：[savepoints]( https://ci.apache.org/projects/flink/flink-docs-release-1.9/zh/ops/state/savepoints.html )
|
||||||
|
|
||||||
|
## 四、状态后端
|
||||||
|
|
||||||
|
### 4.1 状态管理器分类
|
||||||
|
|
||||||
|
默认情况下，所有的状态都存储在 JVM 的堆内存中，在状态数据过多的情况下，这种方式很有可能导致内存溢出，因此 Flink 该提供了其它方式来存储状态数据，这些存储方式统一称为状态后端 (或状态管理器)：
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
主要有以下三种：
|
||||||
|
|
||||||
|
#### 1. MemoryStateBackend
|
||||||
|
|
||||||
|
默认的方式，即基于 JVM 的堆内存进行存储，主要适用于本地开发和调试。
|
||||||
|
|
||||||
|
#### 2. FsStateBackend
|
||||||
|
|
||||||
|
基于文件系统进行存储，可以是本地文件系统，也可以是 HDFS 等分布式文件系统。 需要注意而是虽然选择使用了 FsStateBackend ，但正在进行的数据仍然是存储在 TaskManager 的内存中的，只有在 checkpoint 时，才会将状态快照写入到指定文件系统上。
|
||||||
|
|
||||||
|
#### 3. RocksDBStateBackend
|
||||||
|
|
||||||
|
RocksDBStateBackend 是 Flink 内置的第三方状态管理器，采用嵌入式的 key-value 型数据库 RocksDB 来存储正在进行的数据。等到 checkpoint 时，再将其中的数据持久化到指定的文件系统中，所以采用 RocksDBStateBackend 时也需要配置持久化存储的文件系统。之所以这样做是因为 RocksDB 作为嵌入式数据库安全性比较低，但比起全文件系统的方式，其读取速率更快；比起全内存的方式，其存储空间更大，因此它是一种比较均衡的方案。
|
||||||
|
|
||||||
|
### 4.2 配置方式
|
||||||
|
|
||||||
|
Flink 支持使用两种方式来配置后端管理器：
|
||||||
|
|
||||||
|
**第一种方式**：基于代码方式进行配置，只对当前作业生效：
|
||||||
|
|
||||||
|
```java
|
||||||
|
// 配置 FsStateBackend
|
||||||
|
env.setStateBackend(new FsStateBackend("hdfs://namenode:40010/flink/checkpoints"));
|
||||||
|
// 配置 RocksDBStateBackend
|
||||||
|
env.setStateBackend(new RocksDBStateBackend("hdfs://namenode:40010/flink/checkpoints"));
|
||||||
|
```
|
||||||
|
|
||||||
|
配置 RocksDBStateBackend 时，需要额外导入下面的依赖：
|
||||||
|
|
||||||
|
```xml
|
||||||
|
<dependency>
|
||||||
|
<groupId>org.apache.flink</groupId>
|
||||||
|
<artifactId>flink-statebackend-rocksdb_2.11</artifactId>
|
||||||
|
<version>1.9.0</version>
|
||||||
|
</dependency>
|
||||||
|
```
|
||||||
|
|
||||||
|
**第二种方式**：基于 `flink-conf.yaml` 配置文件的方式进行配置，对所有部署在该集群上的作业都生效：
|
||||||
|
|
||||||
|
```yaml
|
||||||
|
state.backend: filesystem
|
||||||
|
state.checkpoints.dir: hdfs://namenode:40010/flink/checkpoints
|
||||||
|
```
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
> 注：本篇文章所有示例代码下载地址：[flink-state-management]( https://github.com/heibaiying/BigData-Notes/tree/master/code/Flink/flink-state-management)
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
## 参考资料
|
||||||
|
|
||||||
|
+ [Working with State](https://ci.apache.org/projects/flink/flink-docs-release-1.9/dev/stream/state/state.html)
|
||||||
|
+ [Checkpointing](https://ci.apache.org/projects/flink/flink-docs-release-1.9/dev/stream/state/checkpointing.html)
|
||||||
|
+ [Savepoints](https://ci.apache.org/projects/flink/flink-docs-release-1.9/ops/state/savepoints.html#savepoints)
|
||||||
|
+ [State Backends](https://ci.apache.org/projects/flink/flink-docs-release-1.9/ops/state/state_backends.html)
|
||||||
|
+ Fabian Hueske , Vasiliki Kalavri . 《Stream Processing with Apache Flink》. O'Reilly Media . 2019-4-30
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
BIN
pictures/flink-api-stack.png
Normal file
BIN
pictures/flink-api-stack.png
Normal file
Binary file not shown.
|
After Width: | Height: | Size: 22 KiB |
BIN
pictures/flink-bounded-unbounded.png
Normal file
BIN
pictures/flink-bounded-unbounded.png
Normal file
Binary file not shown.
|
After Width: | Height: | Size: 172 KiB |
BIN
pictures/flink-checkpoints-backend.png
Normal file
BIN
pictures/flink-checkpoints-backend.png
Normal file
Binary file not shown.
|
After Width: | Height: | Size: 30 KiB |
BIN
pictures/flink-keyed-state.png
Normal file
BIN
pictures/flink-keyed-state.png
Normal file
Binary file not shown.
|
After Width: | Height: | Size: 131 KiB |
BIN
pictures/flink-on-yarn-session.jpg
Normal file
BIN
pictures/flink-on-yarn-session.jpg
Normal file
Binary file not shown.
|
After Width: | Height: | Size: 88 KiB |
BIN
pictures/flink-operator-state-para1.png
Normal file
BIN
pictures/flink-operator-state-para1.png
Normal file
Binary file not shown.
|
After Width: | Height: | Size: 24 KiB |
BIN
pictures/flink-operator-state-para2.png
Normal file
BIN
pictures/flink-operator-state-para2.png
Normal file
Binary file not shown.
|
After Width: | Height: | Size: 24 KiB |
BIN
pictures/flink-operator-state.png
Normal file
BIN
pictures/flink-operator-state.png
Normal file
Binary file not shown.
|
After Width: | Height: | Size: 197 KiB |
BIN
pictures/flink-process.png
Normal file
BIN
pictures/flink-process.png
Normal file
Binary file not shown.
|
After Width: | Height: | Size: 63 KiB |
BIN
pictures/flink-stack.png
Normal file
BIN
pictures/flink-stack.png
Normal file
Binary file not shown.
|
After Width: | Height: | Size: 35 KiB |
BIN
pictures/flink-standalone-cluster.jpg
Normal file
BIN
pictures/flink-standalone-cluster.jpg
Normal file
Binary file not shown.
|
After Width: | Height: | Size: 72 KiB |
BIN
pictures/flink-state-management.png
Normal file
BIN
pictures/flink-state-management.png
Normal file
Binary file not shown.
|
After Width: | Height: | Size: 24 KiB |
BIN
pictures/flink-stateful-stream.png
Normal file
BIN
pictures/flink-stateful-stream.png
Normal file
Binary file not shown.
|
After Width: | Height: | Size: 221 KiB |
BIN
pictures/flink-stream-barriers.png
Normal file
BIN
pictures/flink-stream-barriers.png
Normal file
Binary file not shown.
|
After Width: | Height: | Size: 12 KiB |
BIN
pictures/flink-subtask-slots.png
Normal file
BIN
pictures/flink-subtask-slots.png
Normal file
Binary file not shown.
|
After Width: | Height: | Size: 67 KiB |
BIN
pictures/flink-task-parallelism.png
Normal file
BIN
pictures/flink-task-parallelism.png
Normal file
Binary file not shown.
|
After Width: | Height: | Size: 291 KiB |
BIN
pictures/flink-task-subtask.png
Normal file
BIN
pictures/flink-task-subtask.png
Normal file
Binary file not shown.
|
After Width: | Height: | Size: 69 KiB |
BIN
pictures/flink-tasks-slots.png
Normal file
BIN
pictures/flink-tasks-slots.png
Normal file
Binary file not shown.
|
After Width: | Height: | Size: 34 KiB |
Loading…
x
Reference in New Issue
Block a user