Maciej Bąk
02/18/2025, 7:53 AMClusterBuilder clusterBuilder = buildCassandraCluster();
CassandraSource<HistoryReadout> historyReadoutCS = new CassandraSource<>(
clusterBuilder,
HistoryReadout.class,
String.format("SELECT * FROM %s.data;", keyspace),
() -> new Mapper.Option[]{Mapper.Option.saveNullFields(true)}
);
then:
DataStreamSource<HistoryReadout> cassHistReadoutSrc =
env.fromSource(historyReadoutCS, WatermarkStrategy.noWatermarks(), "CassHistReadoutSrc");
and:
SingleOutputStreamOperator<Tuple2<String, HistoryReadout>> processedStream = cassHistReadoutSrc.process(new ProcessFunction<HistoryReadout, Tuple2<String, HistoryReadout>>() {
@Override
public void processElement(HistoryReadout readout, ProcessFunction<HistoryReadout, Tuple2<String, HistoryReadout>>.Context ctx, Collector<Tuple2<String, HistoryReadout>> out) throws Exception {
System.out.println("In process element" + readout);
try {
String tableName = HistoryReadoutCassandraServiceUtil.getRowTableName(keyspace, readout);
System.out.println(tableName);
out.collect(new Tuple2<>(tableName, readout));
} catch (Exception e) {
System.err.printf("Error generating table name for record: %s, Error: %s%n", readout, e.getMessage());
}
}
});
at the end:
processedStream.addSink(new SinkFunction<>() {
private transient Session cassandraSession;
@Override
public void invoke(Tuple2<String, HistoryReadout> value, Context context) {
String tableName = value.f0;
HistoryReadout readout = value.f1;
if (cassandraSession == null) {
cassandraSession = createSinkCassandraSession();
}
try {
System.out.printf("Inserting into table: %s, Record: %s%n", tableName, getLogStr(readout));
HistoryReadoutCassandraServiceUtil.processRecordInsert(cassandraSession, tableName, readout);
} catch (Exception e) {
System.err.printf("Error processing record insert: %s, Error: %s%n", readout, e.getMessage());
}
}
private Session createSinkCassandraSession() {
ClusterBuilder clusterBuilder = buildCassandraCluster();
return clusterBuilder.getCluster().connect();
}
private String getLogStr(HistoryReadout readout) {
return readout.tenantid + "-" + readout.iodeviceid + "-" + readout.discriminator + "-" + readout.createdat + "-" + readout.iodevicetype + "-" + readout.serializerid;
}
});
env.execute("Flink Cassandra Connector Job");
}
and this code works if table is small, but for bigger tables I see only:
com.datastax.driver.core.SessionManager$State@57666b8
com.datastax.driver.core.SessionManager$State@74ca0805
com.datastax.driver.core.SessionManager$State@228b352f
com.datastax.driver.core.SessionManager$State@642b3042
com.datastax.driver.core.SessionManager$State@1a5f3752
and nothing happens for a couple of minutes I wonder why, after few minutes I see stream is running, but I am testing on dev environment, in prod I will need to mirgate 2 TB table I need to bne sure that this code is good and does not for example fetch whole table into memory or smth