Tuyển Nguyễn Thanh
03/18/2025, 4:37 AMPedro Mázala
03/18/2025, 9:32 AMlog.retention.ms and log.retention.bytesTuyển Nguyễn Thanh
03/18/2025, 9:50 AMPedro Mázala
03/18/2025, 10:16 AMTuyển Nguyễn Thanh
03/18/2025, 10:18 AMTuyển Nguyễn Thanh
03/18/2025, 10:21 AMKafkaSink<String> prometheusTextSink = KafkaSink.<String>builder()
.setBootstrapServers(KAFKA_BOOTSTRAP_SERVERS)
.setRecordSerializer(KafkaRecordSerializationSchema.builder()
.setTopic(KAFKA_PROMETHEUS_TEXT_SINK_TOPIC)
.setValueSerializationSchema(new SimpleStringSchema())
.build())
.setKafkaProducerConfig(kakfaProps)
.setDeliveryGuarantee(KAFK_DELIVERY_GUARANTEE_SINK)
.setTransactionalIdPrefix("streaming-metrics")
.setProperty("transaction.timeout.ms", "60000")
.build();
KafkaSink<String> errorPrometheusTextSink = KafkaSink.<String>builder()
.setBootstrapServers(KAFKA_BOOTSTRAP_SERVERS)
.setRecordSerializer(KafkaRecordSerializationSchema.builder()
.setTopic(KAFKA_ERROR_PROMETHEUS_TEXT_SINK_TOPIC)
.setValueSerializationSchema(new SimpleStringSchema())
.build())
.setKafkaProducerConfig(kakfaProps)
.setDeliveryGuarantee(KAFK_DELIVERY_GUARANTEE_SINK)
.setTransactionalIdPrefix("streaming-metrics")
.setProperty("transaction.timeout.ms", "60000")
.build();
KafkaSink<byte[]> prometheusRemoteKafkaSink = KafkaSink.<byte[]>builder()
.setBootstrapServers(KAFKA_BOOTSTRAP_SERVERS)
.setRecordSerializer(KafkaRecordSerializationSchema.builder()
.setTopic(KAFKA_PROMETHEUS_REMOTE_WRITE_SINK_TOPIC)
.setValueSerializationSchema(new SerializationSchema<byte[]>() {
public byte[] serialize(byte[] element) {
return element;
};
})
.build())
.setKafkaProducerConfig(kakfaProps)
.setDeliveryGuarantee(KAFK_DELIVERY_GUARANTEE_SINK)
.setTransactionalIdPrefix("streaming-metrics")
.setProperty("transaction.timeout.ms", "60000")
.build();Tuyển Nguyễn Thanh
03/18/2025, 10:22 AMPedro Mázala
03/18/2025, 10:25 AMAT_LEAST_ONCE config. I think it's likely to be something related to a transaction. Even more after seeing your transaction.timeout.ms is 60s.Tuyển Nguyễn Thanh
03/18/2025, 10:26 AMTuyển Nguyễn Thanh
03/18/2025, 10:33 AMTuyển Nguyễn Thanh
03/18/2025, 10:33 AMTuyển Nguyễn Thanh
03/18/2025, 10:33 AMTuyển Nguyễn Thanh
03/18/2025, 10:34 AMTuyển Nguyễn Thanh
03/18/2025, 12:38 PMPedro Mázala
03/18/2025, 12:40 PM