George Leonard
07/05/2024, 3:52 PMCREATE TABLE avro_salespayments (
INVOICENUMBER STRING,
FINTRANSACTIONID STRING,
PAYDATETIME STRING,
PAYTIMESTAMP STRING,
PAID DOUBLE,
PAYTIMESTAMP_WM AS TO_TIMESTAMP(FROM_UNIXTIME(CAST(PAYTIMESTAMP AS BIGINT) / 1000)),
WATERMARK FOR PAYTIMESTAMP_WM AS PAYTIMESTAMP_WM
) WITH (
'connector' = 'kafka',
'topic' = 'avro_salespayments',
'properties.bootstrap.servers' = 'broker:29092',
'scan.startup.mode' = 'earliest-offset',
'properties.group.id' = 'testGroup',
'value.format' = 'avro-confluent',
'value.avro-confluent.schema-registry.url' = 'http://schema-registry:8081',
'value.fields-include' = 'EXCEPT_KEY'
);
with the below the invoicenumber field is null
CREATE TABLE avro_salespayments (
INVOICENUMBER STRING,
FINTRANSACTIONID STRING,
PAYDATETIME STRING,
PAYTIMESTAMP STRING,
PAID DOUBLE,
PAYTIMESTAMP_WM AS TO_TIMESTAMP(FROM_UNIXTIME(CAST(PAYTIMESTAMP AS BIGINT) / 1000)),
WATERMARK FOR PAYTIMESTAMP_WM AS PAYTIMESTAMP_WM
) WITH (
'connector' = 'kafka',
'topic' = 'avro_salespayments',
'properties.bootstrap.servers' = 'broker:29092',
'scan.startup.mode' = 'earliest-offset',
'properties.group.id' = 'testGroup',
'key.format' = 'raw',
'key.fields' = 'INVOICENUMBER',
'value.format' = 'avro-confluent',
'value.avro-confluent.schema-registry.url' = 'http://schema-registry:8081',
'value.fields-include' = 'EXCEPT_KEY'
);
My thinking is I would need a key as I want to join this topic on invoicenumber to avro_salesbaskets, which would be similar key'd.
please advise.