Mahesh Gupta
10/23/2024, 7:38 AM1
2
3
4
5
Then I stopped the application and ingested more records while the job was down:
Second input
1 <-- old data(First input)
2 <-- old data
3 <-- old data
4 <-- old data
5 <-- old data
a <-- new data
b <-- new data
c <-- new data
d <-- new data
However, when I restarted the job, both sets of offsets were consumed, leading to duplicated records in the output:
expected is
a
b
c
d
but it is giving (old + new )
1
2
3
4
5
a
b
c
d
Code snippet
public class FlinkKafkaSchemaMappingExample {
private static final Logger LOG = LoggerFactory.getLogger(FlinkKafkaSchemaMappingExample.class);
public static StreamExecutionEnvironment env;
public static void main(String[] args) throws Exception {
Configuration localConfig = new Configuration();
localConfig.setBoolean(ConfigConstants.LOCAL_START_WEBSERVER, true);
env = StreamExecutionEnvironment.createLocalEnvironmentWithWebUI(localConfig);
// Enable checkpointing (required for exactly-once semantics)
env.enableCheckpointing(10000); // Checkpoint every 10 seconds
env.disableOperatorChaining();
env.getCheckpointConfig().setCheckpointingMode(CheckpointingMode.EXACTLY_ONCE);
env.getCheckpointConfig().enableExternalizedCheckpoints(CheckpointConfig.ExternalizedCheckpointCleanup.RETAIN_ON_CANCELLATION);
CheckpointConfig checkpointConfig = env.getCheckpointConfig();
checkpointConfig.setMinPauseBetweenCheckpoints(1000); // Minimum pause between checkpoints
checkpointConfig.setCheckpointStorage(new FileSystemCheckpointStorage("file:///Users/maheshgupta/Documents/flink_kafka/checkpoints"));
Properties consumerProperties = new Properties();
consumerProperties.setProperty("isolation.level", "read_committed"); // Important for exactly-once
// Set up the Kafka source
KafkaSource<String> kafkaSource = KafkaSource.<String>builder()
.setBootstrapServers("localhost:9092")
.setTopics("inputTopic")
.setGroupId("flink-consumer-group") // Make sure this is unique for this job
.setStartingOffsets(OffsetsInitializer.committedOffsets(OffsetResetStrategy.LATEST))
.setProperty("commit.offsets.on.checkpoint","true")
.setProperties(consumerProperties)
.setValueOnlyDeserializer(new SimpleStringSchema()) // Deserialize the value
.build();
// Generate a unique transactional ID prefix
String transactionalIdPrefix = "flink-transactional-id-" + System.currentTimeMillis();
LOG.info("Using transactional ID prefix: {}", transactionalIdPrefix); // Log the transactional ID prefix
// Set up the Kafka sink
KafkaSink<String> kafkaSink = KafkaSink.<String>builder()
.setBootstrapServers("localhost:9092")
.setRecordSerializer(KafkaRecordSerializationSchema.builder()
.setTopic("outputTopic")
.setValueSerializationSchema(new SimpleStringSchema())
.build())
.setTransactionalIdPrefix(transactionalIdPrefix) // Unique ID for transactions
.setProperty("acks", "all") // Ensure all replicas acknowledge
.setProperty("enable.idempotence", "true") // Idempotence for exactly-once
.setDeliveryGuarantee(DeliveryGuarantee.EXACTLY_ONCE) // or AT_LEAST_ONCE, NONE
.setProperty("transaction.timeout.ms", 10000 * 5 +"")
.build();
// Read from Kafka
DataStream<String> inputStream = env.fromSource(kafkaSource, WatermarkStrategy.noWatermarks(), "Kafka Source");
inputStream.map(new MapFunction<String, String>() {
@Override
public String map(String s) throws Exception {
System.out.println(s); // For debugging
return s;
}
}).sinkTo(kafkaSink);
// Execute the Flink job
env.execute("Flink Kafka Schema Mapping Example");
}
}
Could someone please help me identify if I have any misconfigurations or if there's a missing step? I'm aiming for exactly-once processing and want to avoid duplicate consumption after job restarts.