Mikhail Spirin
06/21/2023, 1:28 PMJoe Karayusuf
06/21/2023, 1:32 PMmaxTimeInBufferMs setting to handle this case.
maxTimeInBufferMS the maximum amount of time an element may remain in the buffer. In most cases elements are flushed as a result of the batch size (in bytes or number) being reached or during a snapshot. However, there are scenarios where an element may remain in the buffer forever or a long period of time. To mitigate this, a timer is constantly active in the buffer such that: while the buffer is not empty, it will flush every maxTimeInBufferMS milliseconds.
Joe Karayusuf
06/21/2023, 1:33 PMKinesisStreamSinkMikhail Spirin
06/21/2023, 1:36 PMAhmed Hamdy
06/21/2023, 1:53 PMMikhail Spirin
06/21/2023, 1:55 PMMikhail Spirin
06/21/2023, 1:57 PMsink.http-client.read-timeout
sink.http-client.protocol.version
sink.batch.max-size
sink.requests.max-inflight
sink.requests.max-buffered
sink.flush-buffer.size
sink.flush-buffer.timeout
sink.fail-on-error
and
kds_sink = KinesisStreamsSink.builder() \
.set_kinesis_client_properties(sink_properties) \ # Required
.set_serialization_schema(SimpleStringSchema()) \ # Required
.set_partition_key_generator(PartitionKeyGenerator.fixed()) \ # Required
.set_stream_name("your-stream-name") \ # Required
.set_fail_on_error(False) \ # Optional
.set_max_batch_size(500) \ # Optional
.set_max_in_flight_requests(50) \ # Optional
.set_max_buffered_requests(10000) \ # Optional
.set_max_batch_size_in_bytes(5 * 1024 * 1024) \ # Optional
.set_max_time_in_buffer_ms(5000) \ # Optional
.set_max_record_size_in_bytes(1 * 1024 * 1024) \ # Optional
.build()
but unfortunately doesn’t work in my case….Ahmed Hamdy
06/21/2023, 1:58 PMMaxTimeInBufferMS using sink.flush-buffer.timeout
https://nightlies.apache.org/flink/flink-docs-master/docs/connectors/table/kinesis/#sink-flush-buffer-timeoutAhmed Hamdy
06/21/2023, 1:59 PMMikhail Spirin
06/21/2023, 2:01 PMMikhail Spirin
06/21/2023, 2:02 PMAhmed Hamdy
06/21/2023, 2:03 PMMikhail Spirin
06/21/2023, 2:04 PMAhmed Hamdy
06/21/2023, 2:12 PMMikhail Spirin
06/21/2023, 2:18 PMWITH (
'connector' = 'kinesis',
'aws.region' = 'us-west-2',
'stream' = 'flink-eventstreamer-poc-prod',
'format' = 'json',
'sink.batch.max-size' = '1',
'sink.requests.max-buffered' = '2',
'sink.flush-buffer.timeout' = '5000'
);Mikhail Spirin
06/21/2023, 2:19 PMAhmed Hamdy
06/21/2023, 2:31 PMMikhail Spirin
06/21/2023, 2:36 PMCREATE TEMPORARY TABLE kafka_source (
...
) WITH (
'connector' = 'kafka',
'topic' = 'flink-eventstreamer-poc-dev',
'scan.startup.mode' = 'latest-offset',
'format' = 'json',
'json.fail-on-missing-field' = 'false',
'json.ignore-parse-errors' = 'true'
);
CREATE TABLE kinesis_sink (
...
) PARTITIONED BY ( ... )
WITH (
'connector' = 'kinesis',
'aws.region' = 'us-west-2',
'stream' = 'flink-eventstreamer-poc-prod',
'format' = 'json',
'sink.batch.max-size' = '5',
'sink.requests.max-buffered' = '10',
'sink.flush-buffer.timeout' = '10000'
);
INSERT INTO kinesis_sink
SELECT
...
FROM kafka_source;Mikhail Spirin
06/21/2023, 3:11 PMCREATE TEMPORARY TABLE kafka_source (
a STRING,
ts TIMESTAMP(3),
WATERMARK FOR ts AS ts - INTERVAL '15' SECOND
) WITH (
'connector' = 'kafka',
...
'scan.startup.mode' = 'latest-offset',
'format' = 'json',
'json.fail-on-missing-field' = 'false',
'json.ignore-parse-errors' = 'true'
);
CREATE TEMPORARY TABLE enrichment_kafka_source (
b INT,
a STRING PRIMARY KEY
) WITH (
'connector' = 'upsert-kafka',
...
'key.format' = 'raw',
'value.format' = 'json',
'properties.auto.offset.reset' = 'earliest'
);
SELECT
e.b AS b,
s.a AS a
FROM
kafka_source AS s
LEFT JOIN enrichment_kafka_source FOR SYSTEM_TIME AS OF f.ts AS e
ON s.a = e.a;Ahmed Hamdy
06/21/2023, 3:20 PMMikhail Spirin
06/21/2023, 3:25 PMAhmed Hamdy
06/21/2023, 5:10 PMMikhail Spirin
06/22/2023, 10:16 AM