George Leonard
05/03/2025, 9:19 AMCREATE OR REPLACE TABLE hive_catalog.iot.factory_iot_north (
ts BIGINT,
metadata ROW<
siteId INTEGER,
deviceId INTEGER,
sensorId INTEGER,
unit STRING,
ts_human STRING,
location ROW<
latitude DOUBLE,
longitude DOUBLE
>,
deviceType STRING
>,
measurement DOUBLE,
event_time AS TO_TIMESTAMP_LTZ(ts, 3),
WATERMARK FOR event_time AS event_time - INTERVAL '5' SECOND
) WITH (
'connector' = 'kafka',
'topic' = 'factory_iot_north',
'properties.bootstrap.servers' = 'broker:29092',
'properties.group.id' = 'devlab0',
'scan.startup.mode' = 'earliest-offset',
'format' = 'json',
'json.fail-on-missing-field' = 'false',
'json.ignore-parse-errors' = 'true'
);
Target Table
CREATE OR REPLACE TABLE fluss_catalog.fluss.factory_iot_north (
ts BIGINT,
metadata ROW<
siteId INTEGER,
deviceId INTEGER,
sensorId INTEGER,
unit STRING,
ts_human STRING,
location ROW<
latitude DOUBLE,
longitude DOUBLE
>,
deviceType STRING
>,
measurement DOUBLE,
partition_month STRING -- must be provided by upstream or insert logic
) PARTITIONED BY (partition_month) WITH (
'bucket.num' = '3'
,'table.datalake.enabled' = 'true'
,'table.auto-partition.time-unit' = 'MONTH'
,'table.auto-partition.num-retention'= '60'
);
Insert statement
Tried both of the below... first one is accepted but flink jobmanager logs contain the below error
INSERT INTO fluss_catalog.fluss.factory_iot_north
SELECT
ts AS ts,
metadata AS metadata,
measurement AS measurement,
DATE_FORMAT(event_time, 'yyyyMM') AS partition_month
FROM hive_catalog.iot.factory_iot_north;
err
2025-05-03 08:21:34,883 INFO org.apache.flink.runtime.executiongraph.ExecutionGraph [] - Source: factory_iot_north[1] -> Calc[2] -> factory_iot_north[3]: Writer (2/2) (18c94214edcecb9d7707c5e3843d960a_cbc357ccb763df2852fee8c4fc7d55f2_1_8) switched from INITIALIZING to FAILED on 172.18.0.12:44331-ea5bc2 @ pipeline-taskmanager-1.pipeline (dataPort=44975).
java.lang.IllegalArgumentException: Currently, Data type 'ROW<`siteId` INT, `deviceId` INT, `sensorId` INT, `unit` STRING, `ts_human` STRING, `location` ROW<`latitude` DOUBLE, `longitude` DOUBLE>, `deviceType` STRING>' is not supported in indexedRow
at com.alibaba.fluss.row.indexed.IndexedRow.isFixedLength(IndexedRow.java:265) ~[fluss-connector-flink-0.6.0.jar:0.6.0]
at com.alibaba.fluss.row.indexed.IndexedRow.calculateVariableColumnLengthListSize(IndexedRow.java:148) ~[fluss-connector-flink-0.6.0.jar:0.6.0]
at com.alibaba.fluss.row.indexed.IndexedRowWriter.<init>(IndexedRowWriter.java:61) ~[fluss-connector-flink-0.6.0.jar:0.6.0]
at com.alibaba.fluss.row.encode.IndexedRowEncoder.<init>(IndexedRowEncoder.java:45) ~[fluss-connector-flink-0.6.0.jar:0.6.0]
at com.alibaba.fluss.row.encode.IndexedRowEncoder.<init>(IndexedRowEncoder.java:38) ~[fluss-connector-flink-0.6.0.jar:0.6.0]
at com.alibaba.fluss.client.table.writer.AppendWriterImpl.<init>(AppendWriterImpl.java:65) ~[fluss-connector-flink-0.6.0.jar:0.6.0]
at com.alibaba.fluss.client.table.writer.TableAppend.createWriter(TableAppend.java:45) ~[fluss-connector-flink-0.6.0.jar:0.6.0]
at com.alibaba.fluss.connector.flink.sink.writer.AppendSinkWriter.initialize(AppendSinkWriter.java:47) ~[fluss-connector-flink-0.6.0.jar:0.6.0]
at com.alibaba.fluss.connector.flink.sink.FlinkSink.createWriter(FlinkSink.java:59) ~[fluss-connector-flink-0.6.0.jar:0.6.0]
at org.apache.flink.streaming.runtime.operators.sink.StatelessSinkWriterStateHandler.createWriter(StatelessSinkWriterStateHandler.java:39) ~[flink-dist-1.20.1.jar:1.20.1]
at org.apache.flink.streaming.runtime.operators.sink.SinkWriterOperator.initializeState(SinkWriterOperator.java:163) ~[flink-dist-1.20.1.jar:1.20.1]
at org.apache.flink.streaming.api.operators.StreamOperatorStateHandler.initializeOperatorState(StreamOperatorStateHandler.java:147) ~[flink-dist-1.20.1.jar:1.20.1]
at org.apache.flink.streaming.api.operators.AbstractStreamOperator.initializeState(AbstractStreamOperator.java:294) ~[flink-dist-1.20.1.jar:1.20.1]
at org.apache.flink.streaming.runtime.tasks.RegularOperatorChain.initializeStateAndOpenOperators(RegularOperatorChain.java:106) ~[flink-dist-1.20.1.jar:1.20.1]
at org.apache.flink.streaming.runtime.tasks.StreamTask.restoreStateAndGates(StreamTask.java:858) ~[flink-dist-1.20.1.jar:1.20.1]
at org.apache.flink.streaming.runtime.tasks.StreamTask.lambda$restoreInternal$5(StreamTask.java:812) ~[flink-dist-1.20.1.jar:1.20.1]
at org.apache.flink.streaming.runtime.tasks.StreamTaskActionExecutor$1.call(StreamTaskActionExecutor.java:55) ~[flink-dist-1.20.1.jar:1.20.1]
at org.apache.flink.streaming.runtime.tasks.StreamTask.restoreInternal(StreamTask.java:812) ~[flink-dist-1.20.1.jar:1.20.1]
at org.apache.flink.streaming.runtime.tasks.StreamTask.restore(StreamTask.java:771) ~[flink-dist-1.20.1.jar:1.20.1]
at org.apache.flink.runtime.taskmanager.Task.runWithSystemExitMonitoring(Task.java:970) ~[flink-dist-1.20.1.jar:1.20.1]
at org.apache.flink.runtime.taskmanager.Task.restoreAndInvoke(Task.java:939) ~[flink-dist-1.20.1.jar:1.20.1]
at org.apache.flink.runtime.taskmanager.Task.doRun(Task.java:763) ~[flink-dist-1.20.1.jar:1.20.1]
at org.apache.flink.runtime.taskmanager.Task.run(Task.java:575) ~[flink-dist-1.20.1.jar:1.20.1]
at java.lang.Thread.run(Unknown Source) ~[?:?]
2
and
INSERT INTO fluss_catalog.fluss.factory_iot_north
SELECT
ts
,metadata.siteId
,metadata.deviceId
,metadata.sensorId
,metadata.unit
,metadata.ts_human
,metadata.location.latitude
,metadata.location.longitude
,metadata.deviceType
,measurement
,DATE_FORMAT(event_time, 'yyyyMM')
FROM hive_catalog.iot.factory_iot_north;
Err, ps the above select works... it's some how not happy with the source/target table mapping
Caused by: org.apache.flink.table.api.ValidationException: Column types of query result and sink for 'fluss_catalog.fluss.factory_iot_north' do not match.
Cause: Different number of columns.
Query schema: [ts: BIGINT, siteId: INT, deviceId: INT, sensorId: INT, unit: STRING, ts_human: STRING, latitude: DOUBLE, longitude: DOUBLE, deviceType: STRING, measurement: DOUBLE, EXPR$10: STRING]
Sink schema: [ts: BIGINT, metadata: ROW<`siteId` INT, `deviceId` INT, `sensorId` INT, `unit` STRING, `ts_human` STRING, `location` ROW<`latitude` DOUBLE, `longitude` DOUBLE>, `deviceType` STRING>, measurement: DOUBLE, partition_month: STRING]