kapil singh
08/01/2023, 7:51 AMfrom pyflink.common.serialization import SimpleStringSchema
from pyflink.common.typeinfo import Types
from pyflink.common.watermark_strategy import WatermarkStrategy
from pyflink.datastream import StreamExecutionEnvironment
from pyflink.datastream.connectors.kafka import KafkaSource, KafkaSink, KafkaOffsetsInitializer, KafkaRecordSerializationSchema, DeliveryGuarantee
def kafkaread():
env = StreamExecutionEnvironment.get_execution_environment()
brokers = 'cluster-0-kafka-bootstrap.strimzi.svc.cluster.local:9092'
source = KafkaSource.builder()\
.set_bootstrap_servers(brokers) \
.set_topics("locations") \
.set_group_id("flinktestpers") \
.set_starting_offsets(KafkaOffsetsInitializer.earliest()) \
.set_value_only_deserializer(SimpleStringSchema()) \
.build()
ds = env.from_source(source, WatermarkStrategy.no_watermarks(), "Kafka Source")
ds.print()
# Define the Kafka sink
sink = KafkaSink.builder() \
.set_bootstrap_servers(brokers) \
.set_record_serializer(
KafkaRecordSerializationSchema.builder()
.set_topic("flinksink") # Set your output topic here
.set_value_serialization_schema(SimpleStringSchema())
.build()
) \
.set_delivery_guarantee(DeliveryGuarantee.AT_LEAST_ONCE) \
.build()
ds.sink_to(sink)
env.execute('kafkaread')
if __name__ == '__main__':
kafkaread()
FLINK_VERSION=1.16.2Dian Fu
08/02/2023, 11:51 AMkapil singh
08/04/2023, 1:11 PMkapil singh
08/04/2023, 1:11 PM