Aram
06/28/2024, 3:47 PMclass SomeRichMapFunction : RichMapFunction<Data, OutputData>() {
private lateinit var oneFairlyMediumSizedElement: OtherData
override fun open(parameters: Configuration) {
oneFairlyMediumSizedElement = runtimeContext.getBroadcastVariable<OtherData>("oneFairlyMediumSizedElementBroadcastSet").single()
}
override fun map(value: Data): OutputData {
TODO("Do some funky calc with value and oneFairlyMediumSizedElement")
}
}
fun calculate(huge: DataSet<Data>, oneFairlyMediumSizedElement: DataSet<OtherData>) =
huge.map(SomeRichMapFunction())
.withBroadcastSet(oneFairlyMediumSizedElement, "oneFairlyMediumSizedElementBroadcastSet")D. Draco O'Brien
06/28/2024, 5:41 PMD. Draco O'Brien
06/28/2024, 5:43 PMclass SomeBroadcastProcessFunction : BroadcastProcessFunction<Data, OtherData, OutputData>() {
override fun processElement(data: Data, ctx: Context, out: Collector<OutputData>) {
val oneFairlyMediumSizedElement = ctx.getBroadcastState(...).value() // Access the broadcast variable
// TODO: Do some funky calc with data and oneFairlyMediumSizedElement, emit result with out.collect(...)
}
override fun processBroadcastElement(otherData: OtherData, ctx: Context, state: BroadcastState<...>) {
// Update the broadcast state when new elements arrive
state.put("key", otherData) // The key could be anything that identifies this element, adjust as needed
}
}D. Draco O'Brien
06/28/2024, 5:43 PM