Hey there :wave: - I'm a newcomer to the Flink ec...
# troubleshooting
a
Hey there πŸ‘‹ - I'm a newcomer to the Flink ecosystem. I'm attempting to take a DataStream with a DynamoDB source and filter + map + sink it into 30+ different tables. I was wondering if this was the idiomatic way to do this. I'm a bit puzzled at which of the
.execute
calls on both the
StreamTableEnvironment
and
StreamExecutionEnvironment
are actually required. I'm unfortunately pinned to 1.19.1 at the moment.
Copy code
abstract class TableSink<T>() : Serializable {
  abstract fun createDestinationTable()
  abstract fun getTableDescriptor(): TableDescriptor
  abstract fun matchEvent(event: DynamoDBEvent?): Boolean
  abstract fun mapEvent(event: DynamoDBEvent): T
  abstract fun getSchema(): Schema
  abstract fun getDataType(): DataType
  abstract fun getTypeInformation(): TypeInformation<T>
}

fun main() {
  val env = StreamExecutionEnvironment.getExecutionEnvironment()
  val tableEnv: StreamTableEnvironment = StreamTableEnvironment.create(env)
  val sinks = listOf(...a number of TableSinks...)

  val source = DynamoDbStreamsSource.builder<DynamoDBEvent>()
    .setStreamArn(streamArn)
    .setDeserializationSchema(DynamoDBEventDeserializationSchema())
    .build()

  val stream = env.fromSource(
    source,
    WatermarkStrategy
      .forBoundedOutOfOrderness<DynamoDBEvent>(Duration.ofSeconds(20))
    "DynamoDBEvents",
    TypeInformation.of(DynamoDBEvent::class.java)
  )
    .uid(streamArn).name("DynamoDBEvents")

  for (sink in sinks) {
    sink.createDestinationTable()
    
    tableEnv.createTable(sink.tableName, sink.getTableDescriptor())
    val filtered = stream
      .filter { event -> sink.matchEvent(event) }
      .map { event -> sink.mapEvent(event) }
      .uid(sink.tableName).name(sink.tableName)
      .returns(sink.getTypeInformation() as TypeInformation<Any>)

    val tableStream = tableEnv.fromDataStream(filtered, sink.getSchema())
    tableStream.insertInto(sink.tableName).execute()
  }

  env.execute()
}