Hi everyone, I'm trying to create a hybrid table, but it's not working. My realtime table is functioning well, but the offline table isn't receiving any data. below is my schema for the table: { "schemaName": "physio_patient_schema", "dimensionFieldSpecs": [ { "name": "patient_id", "dataType": "INT" }, { "name": "city", "dataType": "STRING" }, { "name": "landmark", "dataType": "STRING" }, { "name": "line1", "dataType": "STRING" }, { "name": "line2", "dataType": "STRING" }, { "name": "state", "dataType": "STRING" }, { "name": "email_address", "dataType": "STRING" }, { "name": "first_name", "dataType": "STRING" }, { "name": "last_name", "dataType": "STRING" }, { "name": "middle_name", "dataType": "STRING" }, { "name": "mobile", "dataType": "STRING" }, { "name": "sex", "dataType": "STRING" } "metricFieldSpecs": [], "dateTimeFieldSpecs": [ { "name": "__source_ts_ms", "dataType": "TIMESTAMP", "format": "1MILLISECONDSEPOCH", "granularity": "1:MILLISECONDS" }, { "name": "created_at", "dataType": "TIMESTAMP", "format": "1MILLISECONDSEPOCH", "granularity": "1:MILLISECONDS", "defaultNullValue": null }, { "name": "poc_created_at", "dataType": "TIMESTAMP", "format": "SIMPLE_DATE_FORMAT|yyyy-MM-dd HHmmss", "granularity": "1:MILLISECONDS", "defaultNullValue": null } ], "timeFieldSpec": null, "primaryKeyColumns": [ "patient_id" ], "invertedIndexColumns": [], "sortedColumn": [] } offline table conf: { _"tableName"_: "physio_patient_schema", _"tableType"_: "OFFLINE", _"segmentsConfig"_: { _"timeColumnName"_: "__source_ts_ms", _"schemaName"_: "physio_patient_schema", _"replication"_: "1", _"replicasPerPartition"_: "1" }, _"ingestionConfig"_: { _"batchIngestionConfig"_: { _"segmentIngestionType"_: "APPEND", _"segmentIngestionFrequency"_: "HOURLY" } }, _"tableIndexConfig"_: { _"loadMode"_: "MMAP" }, _"tenants"_: {}, _"metadata"_: {} } Realtime table conf: { _"tableName"_: "physio_patient_schema", _"tableType"_: "REALTIME", _"segmentsConfig"_: { _"timeColumnName"_: "__source_ts_ms", _"segmentPushType"_: "APPEND", _"segmentAssignmentStrategy"_: "BalanceNumSegmentAssignmentStrategy", _"schemaName"_: "physio_patient_schema", _"replicasPerPartition"_: "1" }, _"tenants"_: {}, _"task"_: { _"taskTypeConfigsMap"_: { _"RealtimeToOfflineSegmentsTask"_: { _"bufferTimePeriod"_: "1m", _"bucketTimePeriod"_: "5m", _"roundBucketTimePeriod"_: "1m", _"schedule"_: "0 * * * * ?", _"mergeType"_: "rollup", _"count.aggregationType"_: "max", _"maxNumRecordsPerSegment"_: "500" } } }, _"tableIndexConfig"_: { _"loadMode"_: "MMAP", _"streamConfigs"_: { _"streamType"_: "kafka", _"stream.kafka.consumer.type"_: "lowlevel", _"stream.kafka.topic.name"_: "topic_physio.physio.patient", _"stream.kafka.decoder.class.name"_: "org.apache.pinot.plugin.stream.kafka.KafkaJSONMessageDecoder", _"stream.kafka.consumer.factory.class.name"_: "org.apache.pinot.plugin.stream.kafka20.KafkaConsumerFactory", _"stream.kafka.broker.list"_: "kafka:9092", _"stream.kafka.consumer.prop.auto.offset.reset"_: "smallest", _"stream.kafka.consumer.prop.enable.auto.commit"_: "false", _"realtime.segment.flush.threshold.rows"_: "500" } }, _"routing"_: { _"instanceSelectorType"_: "strictReplicaGroup" }, _"upsertConfig"_: { _"mode"_: "FULL", _"enableSnapshot"_: true, _"enablePreload"_: true, _"upsertTTL"_: "1h" }, _"metadata"_: { _"customConfigs"_: {} } }

I see a few issues, which probably don't explain your original problem. Nonetheless, are issues to be addressed: 1. I see upsert enabled, it is not supported for offline/hybrid tables. 2. Seems like you are trying to move data every few minutes. I think this might be a understanding gap.