Smaran Balla
03/20/2025, 5:51 AMThere are thousands of segments getting autocommitted with 0 docs like below...
{
"segment.realtime.endOffset": "{\"shardId-000000000000\":\"496xxxx0056588014199503011956563195560534025543652212738\"}",
"segment.start.time": "1742422538998",
"segment.time.unit": "MILLISECONDS",
"segment.flush.threshold.size": "1000000",
"segment.realtime.startOffset": "{\"shardId-000000000000\":\"496xxxx0056588014199503011956563195560534025543652212738\"}",
"segment.end.time": "1742422538998",
"segment.total.docs": "0",
"segment.realtime.numReplicas": "1",
"segment.creation.time": "1742422537567",
"segment.index.version": "v3",
"segment.crc": "2756404785",
"segment.realtime.status": "DONE",
"segment.download.url": "s3://******/controller/********/****__0__1001__20250319T2215Z"
}
Could u pls help??
cc: @Mayank @Anand Kr Shaw @Xiang FuSmaran Balla
03/20/2025, 5:53 AM{
"tableName": "kyc_apps_metadata",
"tableType": "REALTIME",
"tenants": {
"broker": "DefaultTenant",
"server": "KYC",
"tagOverrideConfig": {}
},
"segmentsConfig": {
"schemaName": "kyc_apps_metadata",
"timeColumnName": "eventEmitTime",
"replication": "1",
"retentionTimeUnit": null,
"retentionTimeValue": null,
"completionConfig": null,
"crypterClassName": null,
"peerSegmentDownloadScheme": null
},
"tableIndexConfig": {
"nullHandlingEnabled": true,
"loadMode": "MMAP",
"invertedIndexColumns": [
"platform",
"clientType",
"city",
"pincode",
"appNumber"
],
"createInvertedIndexDuringSegmentGeneration": false,
"rangeIndexColumns": [],
"sortedColumn": [],
"bloomFilterColumns": [],
"bloomFilterConfigs": null,
"noDictionaryColumns": [],
"onHeapDictionaryColumns": [],
"varLengthDictionaryColumns": [],
"enableDefaultStarTree": false,
"starTreeIndexConfigs": null,
"enableDynamicStarTreeCreation": false,
"segmentPartitionConfig": null,
"columnMinMaxValueGeneratorMode": null,
"aggregateMetrics": false,
"streamConfigs": {
"iamRoleBasedAccessEnabled": "true",
"roleArn": "arn:aws:iam::*******:role/*******",
"streamType": "kinesis",
"stream.kinesis.topic.name": "*********",
"region": "*********",
"shardIteratorType": "TRIM_HORIZON",
"stream.kinesis.consumer.type": "lowlevel",
"stream.kinesis.fetch.timeout.millis": "30000",
"stream.kinesis.decoder.class.name": "org.apache.pinot.plugin.stream.kafka.KafkaJSONMessageDecoder",
"stream.kinesis.consumer.factory.class.name": "org.apache.pinot.plugin.stream.kinesis.KinesisConsumerFactory",
"realtime.segment.flush.threshold.rows": "1000000",
"realtime.segment.flush.threshold.time": "24h"
},
"DictionaryColumns": []
},
"metadata": {},
"ingestionConfig": {
"continueOnError": true,
"filterConfig": {
"filterFunction": "NOT STARTSWITH(eventName, 'metadata')"
},
"transformConfigs": [
{
"columnName": "ingestedAt",
"transformFunction": "toDateTime(now(), 'yyyy-MM-dd HH:mm:ss.sss')"
},
{
"columnName": "lastUpdatedAt",
"transformFunction": "toDateTime(now(), 'yyyy-MM-dd HH:mm:ss.sss')"
}
]
},
"quota": {
"storage": null,
"maxQueriesPerSecond": null
},
"task": null,
"routing": {
"instanceSelectorType": "strictReplicaGroup",
"segmentPrunerTypes": [
"partition"
]
},
"query": {
"timeoutMs": null
},
"fieldConfigList": null,
"upsertConfig": {
"mode": "PARTIAL",
"defaultPartialUpsertStrategy": "OVERWRITE",
"partialUpsertStrategies": {
"ingestedAt": "IGNORE"
}
},
"tierConfigs": null,
"isDimTable": false
}Smaran Balla
03/20/2025, 7:47 AMMayank
Xiang Fu
Smaran Balla
03/23/2025, 5:44 PMXiang Fu
RealtimeSegmentValidationManager?Xiang Fu
RealtimeSegmentValidationManager from the swagger ui, it’s under controller periodical task