Pranay Das
05/30/2024, 6:20 AM{
"timestamp": 1717047000567,
"ssp_id": 10,
"dsp_id": 3,
"uuid": "123456-a09a-492b-9694-f52016c8c72c",
"domain": "google.com",
"cat1": 1,
"cat2": 2,
"cat3": 3,
"cat4": 4
}
Code Snippet
DataStream<Segment> uniqueSegments = segments.keyBy(value -> new Segment(
value.uuid,
value.doman,
value.some_id_int,
value.some_ID_int,
value.cat1,
value.cat2,
value.cat3,
value.cat4))
.process(new EventProcessor()).uid("some-uuid");
public class SegmentEventProcessor extends KeyedProcessFunction<Segment, Segment, Segment> {
private static final long serialVersionUID = 1L;
private transient ValueState<Boolean> flagState;
private transient ValueState<Long> timerState;
private static final long ONE_MONTH = 2592000000L;
@Override
public void open(OpenContext openContext) {
ValueStateDescriptor<Boolean> flagDescriptor = new ValueStateDescriptor<>(
"flag",
Types.BOOLEAN);
flagState = getRuntimeContext().getState(flagDescriptor);
ValueStateDescriptor<Long> timerDescriptor = new ValueStateDescriptor<>(
"timer-state",
Types.LONG);
timerState = getRuntimeContext().getState(timerDescriptor);
}
@Override
public void processElement(Segment segment, Context context, Collector<Segment> collector) throws Exception {
if (flagState.value() == null || !flagState.value()) {
collector.collect(segment);
flagState.update(true);
long timer = context.timerService().currentProcessingTime() + ONE_MONTH;
context.timerService().registerProcessingTimeTimer(timer);
timerState.update(timer);
}
}
@Override
public void onTimer(long timestamp, OnTimerContext ctx, Collector<Segment> out) {
timerState.clear();
flagState.clear();
}
}Shan P S
05/30/2024, 8:08 PMkeyBy(value -> new Segment(
value.uuid,
value.doman,
value.some_id_int,
value.some_ID_int,
value.cat1,
value.cat2,
value.cat3,
value.cat4))
If the key is UUID shouldnt it be?
keyBy(value ->
value.uuid)
Even then, if you try to maintain all UUIDs in Flink state for a month, it will be huge.Pranay Das
05/31/2024, 8:08 AMShan P S
06/01/2024, 12:25 AMPranay Das
06/06/2024, 10:24 AM