Alex Morley
02/04/2025, 2:22 PMstandalone mode with the operator but it still just submits job then skips all the rest. We think we got this working before but can't reproduce it.
The use case is to take data from many S3 "partitions" in order and order them before writing the data to kafka (open to alternatives that would achieve the same).
Some (horrible, please excuse) code in 🧵Alex Morley
02/04/2025, 2:22 PMfor (S3Source partition : partitions) {
Configuration config = new Configuration();
StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment(config);
env.setRuntimeMode(RuntimeExecutionMode.BATCH);
env.setRestartStrategy(RestartStrategies.fixedDelayRestart(3, 10000));
StreamTableEnvironment tableEnv = StreamTableEnvironment.create(env);
System.out.println("Processing S3 folder: " + partition.name);
var pipelineName = "plc-backfill-" + partition.name.split("/")[5].split("%")[0];
config.setString("pipeline.name", pipelineName);
System.out.println("11.01 add local jar");
System.out.println("create table");
tableEnv.executeSql("DROP TABLE IF EXISTS parquetSource");
tableEnv.executeSql(("CREATE TABLE parquetSource" + " (\n" +
"`EVENT_STREAM_ID` STRING," +
"`PAYLOAD` STRING," +
"`EVENT_TIMESTAMP` TIMESTAMP(3)," +
"`KAFKA_TIMESTAMP` TIMESTAMP(3)" +
") WITH (" +
"'path' = 's3a://...-$env/$key'," +
"'connector' = 'filesystem'," +
"'format' = 'parquet'" +
")").replaceAll("\\$key", partition.prefix.replaceAll("%20", " ")).replaceAll("\\$env", this.env));
System.out.println("create sink table");
tableEnv.executeSql("DROP TABLE IF EXISTS sinkStream");
tableEnv.executeSql(("CREATE TABLE sinkStream (" +
"`key` STRING," +
"`value` STRING" +
") WITH (" +
"'connector' = 'kafka'," +
"'key.format' = 'raw'," +
"'key.fields' = 'key'," +
...
)
Table tbl = tableEnv.sqlQuery(
"SELECT " +
" CAST( " +
" CONCAT( " +
" '{\"Schema\": {\"type\": \"string\", \"optional\": \"false\"}, \"payload\": \"', " +
" EVENT_STREAM_ID, " +
" '\"}' " +
" ) AS STRING " +
" ) AS `key`, " +
" PAYLOAD AS `value` " +
"FROM parquetSource " +
"ORDER BY KAFKA_TIMESTAMP ASC"
);
try {
System.out.println("Executing job for: " + partition.name);
tableEnv.toDataStream(tbl).sinkTo(KafkaSink.<Row>builder()
.setBootstrapServers(brokerList)
.setRecordSerializer(new MyKafkaRecordSerializationSchema(outputTopic))
.setKafkaProducerConfig(kafkaSinkProps(pipelineName, brokerList))
.setDeliveryGuarantee(DeliveryGuarantee.AT_LEAST_ONCE)
.build());
env.executeAsync(pipelineName);
System.out.println("Close env");
System.out.println("Finished job for" + partition.name);
} catch (Exception e) {
throw new RuntimeException(e);
}