George Leonard
04/30/2025, 7:10 PM{
"ts" : 1729312551000,
"metadata" : {
"siteId" : 1009,
"deviceId" : 1042,
"sensorId" : 10180,
"unit" : "Psi",
"ts_human" : "2024-10-02T00:00:00.869Z",
"location": {
"latitude": -26.195246,
"longitude": 28.034088
},
"deviceType" : "Oil Pump",
},
"measurement" : 1013.3997
}
This is the flink create table i'm creating. idea is to consume from a kafka topic, I'm not using serialisation or a schema registry.
CREATE OR REPLACE TABLE hive_catalog.iot.factory_iot_north (
ts BIGINT,
metadata ROW<
siteId INTEGER,
deviceId INTEGER,
sensorId INTEGER,
unit STRING,
ts_human STRING,
location ROW<
latitude DOUBLE,
longitude DOUBLE
>,
deviceType STRING
>,
measurement DOUBLE
TS_WM AS TO_TIMESTAMP(FROM_UNIXTIME(ts))),
WATERMARK FOR TS_WM AS TS_WM
) WITH (
'connector' = 'kafka',
'topic' = 'factory_iot_north',
'properties.bootstrap.servers' = 'broker:29092',
'properties.group.id' = 'devlab0',
'scan.startup.mode' = 'earliest-offset',
'format' = 'json',
'value.fields-include' = 'ALL'
);
see err.txt attached