Hello guys,
This is my table
{
"REALTIME": {
"tableName": "transaction_REALTIME",
"tableType": "REALTIME",
"segmentsConfig": {
"minimizeDataMovement": false,
"timeColumnName": "updated_at",
"schemaName": "transaction",
"replication": "1"
},
"tenants": {
"broker": "cp_frm",
"server": "cp_frm"
},
"tableIndexConfig": {
"rangeIndexVersion": 2,
"autoGeneratedInvertedIndex": false,
"createInvertedIndexDuringSegmentGeneration": false,
"loadMode": "MMAP",
"streamConfigs": {
"streamType": "pulsar",
"
stream.pulsar.topic.name": "
topic name",
"stream.pulsar.bootstrap.servers":
pulsar server
"stream.pulsar.consumer.prop.auto.offset.reset": "smallest",
"stream.pulsar.consumer.type": "lowlevel",
"
stream.pulsar.decoder.class.name": "org.apache.pinot.plugin.inputformat.avro.SimpleAvroMessageDecoder",
"
stream.pulsar.consumer.factory.class.name": "
schema"
},
"enableDefaultStarTree": false,
"enableDynamicStarTreeCreation": false,
"aggregateMetrics": false,
"nullHandlingEnabled": true,
"columnMajorSegmentBuilderEnabled": true,
"optimizeDictionary": false,
"optimizeDictionaryForMetrics": false,
"optimizeDictionaryType": false,
"noDictionarySizeRatioThreshold": 0.85
},
"metadata": {},
"routing": {
"instanceSelectorType": "strictReplicaGroup"
},
"query": {
"timeoutMs": 3000
},
"instanceAssignmentConfigMap": {
"COMPLETED": {
"tagPoolConfig": {
"tag": "cp_frm_REALTIME",
"poolBased": false,
"numPools": 0
},
"replicaGroupPartitionConfig": {
"replicaGroupBased": true,
"numInstances": 0,
"numReplicaGroups": 1,
"numInstancesPerReplicaGroup": 0,
"numPartitions": 2,
"numInstancesPerPartition": 1,
"minimizeDataMovement": false,
"partitionColumn": "id"
},
"partitionSelector": "INSTANCE_REPLICA_GROUP_PARTITION_SELECTOR",
"minimizeDataMovement": false
},
"CONSUMING": {
"tagPoolConfig": {
"tag": "cp_frm_REALTIME",
"poolBased": false,
"numPools": 0
},
"replicaGroupPartitionConfig": {
"replicaGroupBased": true,
"numInstances": 0,
"numReplicaGroups": 1,
"numInstancesPerReplicaGroup": 0,
"numPartitions": 2,
"numInstancesPerPartition": 1,
"minimizeDataMovement": false,
"partitionColumn": "id"
},
"partitionSelector": "INSTANCE_REPLICA_GROUP_PARTITION_SELECTOR",
"minimizeDataMovement": false
}
},
"upsertConfig": {
"hashFunction": "NONE",
"defaultPartialUpsertStrategy": "OVERWRITE",
"upsertViewRefreshIntervalMs": 3000,
"partialUpsertStrategies": {},
"enableSnapshot": true,
"metadataTTL": 0,
"deletedKeysTTL": 0,
"enablePreload": true,
"consistencyMode": "SNAPSHOT",
"newSegmentTrackingTimeMs": 10000,
"dropOutOfOrderRecord": false,
"enableDeletedKeysCompactionConsistency": false,
"allowPartialUpsertConsumptionDuringCommit": false,
"mode": "PARTIAL"
},
"ingestionConfig": {
"continueOnError": false,
"rowTimeValueCheck": false,
"transformConfigs": [
{
"columnName": "updated_at",
"transformFunction": "NOW()"
}
],
"segmentTimeValueCheck": true
},
"isDimTable": false
}
When data gets inserted during consumption, the data is visible.
Once the segment gets committed the data is visible only when I put SET skipUpsert=true or when I reloaded segements. Can someone help on why this is happening and the solution it.
pinot version - 1.2.0
@Mayank