<@UDSQ3A58W> if you can provide the following info...
# generals
Subbu Subramaniam
02/28/2019, 1:16 AM@User if you can provide the following info that will help a lot:
1. Zookeeper Segment metadata for the consuming segment : Should be in helix-cluster-name/pinot/PROPERTYSTORE/CONFIGS/SEGMENTS/tableName/segmentName
2. TableConfig
Thanks
a
Ananth Packkildurai
02/28/2019, 1:46 AM
one of the segment metadata
Copy code
{
"id": "clog_v3__14__17__20190228T0105Z",
"simpleFields": {
"segment.crc": "1177960103",
"segment.creation.time": "1551315929613",
"segment.end.time": "1551306283884",
"segment.flush.threshold.size": "40000000",
"segment.flush.threshold.time": null,
"segment.index.version": "v3",
"segment.name": "clog_v3__14__17__20190228T0105Z",
"segment.partition.metadata": "{\"columnPartitionMap\":{}}",
"segment.realtime.download.url": "<http://10.0.59.154:9000/segments/clog_v3/clog_v3__14__17__20190228T0105Z>",
"segment.realtime.endOffset": "2708178197",
"segment.realtime.numReplicas": "2",
"segment.realtime.startOffset": "2708157487",
"segment.realtime.status": "DONE",
"segment.start.time": "1551306266581",
"segment.table.name": "clog_v3",
"segment.time.unit": "MILLISECONDS",
"segment.total.docs": "20710",
"segment.type": "REALTIME"
},
"listFields": {},
"mapFields": {}
}Ananth Packkildurai
02/28/2019, 1:46 AM
and table config
Copy code
{
"id":"clog_v3_REALTIME"
,"simpleFields":{
"tableType":"REALTIME"
,"tenants":"{\"broker\":\"DefaultTenant\",\"server\":\"DefaultTenant\",\"tagOverrideConfig\":null}"
,"tableIndexConfig":"{\"streamConfigs\":{\"stream.kafka.decoder.class.name\":\"com.linkedin.pinot.core.realtime.impl.kafka.KafkaJSONMessageDecoder\",\"streamType\":\"kafka\",\"stream.kafka.decoder.prop.schema.registry.rest.url\":\"\",\"realtime.segment.commit.timeoutSeconds\":\"7200\",\"realtime.segment.flush.threshold.size\":\"80000000\",\"stream.kafka.consumer.type\":\"simple\",\"realtime.segment.flush.threshold.time\":\"5m\",\"stream.kafka.zk.broker.url\":\"metrics-kafka-zookeeper3:2181,metrics-kafka-zookeeper1:2181,metrics-kafka-zookeeper2:2181\",\"stream.kafka.broker.list\":\"<http://metrics-kafka-04631307d0a1258a1.nebula.tinyspeck.com:9092,metrics-kafka-0d0d2e024b9e073d8.nebula.tinyspeck.com:9092,metrics-kafka-02d885b15e8e33fbd.nebula.tinyspeck.com:9092,metrics-kafka-0b677a4aaf0e29290.nebula.tinyspeck.com:9092\|metrics-kafka-04631307d0a1258a1.nebula.tinyspeck.com:9092,metrics-kafka-0d0d2e024b9e073d8.nebula.tinyspeck.com:9092,metrics-kafka-02d885b15e8e33fbd.nebula.tinyspeck.com:9092,metrics-kafka-0b677a4aaf0e29290.nebula.tinyspeck.com:9092\>",\"stream.kafka.consumer.prop.auto.offset.reset\":\"largest\",\"stream.kafka.topic.name\":\"clog-json\"},\"segmentPartitionConfig\":null,\"aggregateMetrics\":false,\"invertedIndexColumns\":[\"family\",\"label\",\"team_id\",\"enterprise_team_id\",\"lead_id\",\"app_version\",\"app_release\",\"release_version\"],\"autoGeneratedInvertedIndex\":false,\"createInvertedIndexDuringSegmentGeneration\":false,\"sortedColumn\":[],\"loadMode\":\"mmap\",\"streamConsumptionConfig\":null,\"segmentFormatVersion\":null,\"columnMinMaxValueGeneratorMode\":null,\"noDictionaryColumns\":[\"clog_json\",\"user_id\",\"enterprise_user_id\"],\"noDictionaryConfig\":null,\"onHeapDictionaryColumns\":null,\"starTreeIndexSpec\":null,\"starTreeIndexConfigs\":null}"
,"metadata":"{\"customConfigs\":null}"
,"segmentsConfig":"{\"retentionTimeUnit\":\"HOURS\",\"retentionTimeValue\":\"2\",\"segmentPushFrequency\":\"daily\",\"segmentPushType\":\"APPEND\",\"replication\":\"2\",\"schemaName\":\"clog\",\"timeColumnName\":\"event_timestamp\",\"timeType\":\"MILLISECONDS\",\"segmentAssignmentStrategy\":\"BalanceNumSegmentAssignmentStrategy\",\"replicaGroupStrategyConfig\":null,\"hllConfig\":null,\"replicasPerPartition\":\"2\"}"
,"tableName":"clog_v3_REALTIME"
}
,"listFields":{
}
,"mapFields":{
}
}s
Subbu Subramaniam
03/01/2019, 11:06 PMThe segment metadata for a consuming segment was what i was looking for. If a consuming segment has metadata that has 0 rows (somehow due to a bug) then some of your symptoms are possible.
2 Views