Hi Team :wave:, I created a REALTIME table where i...
# troubleshooting
s
Hi Team 👋, I created a REALTIME table where it consumes data from Kinesis stream, Initially there were 4 shards (shardId-000000000000, shardId-000000000001, shardId-000000000002, shardId-000000000003) and the table has 4 segments which is fine, but when I changed the provisioned shards to 8 for that kinesis stream, • Pinot is not able to consume data from the newly created shards which are (shardId-000000000004, shardId-000000000005, ....shardId-000000000011), this is one problem... • And another problem that I faced was,
There are thousands of segments getting autocommitted with 0 docs
like below...
Copy code
{
  "segment.realtime.endOffset": "{\"shardId-000000000000\":\"496xxxx0056588014199503011956563195560534025543652212738\"}",
  "segment.start.time": "1742422538998",
  "segment.time.unit": "MILLISECONDS",
  "segment.flush.threshold.size": "1000000",
  "segment.realtime.startOffset": "{\"shardId-000000000000\":\"496xxxx0056588014199503011956563195560534025543652212738\"}",
  "segment.end.time": "1742422538998",
  "segment.total.docs": "0",
  "segment.realtime.numReplicas": "1",
  "segment.creation.time": "1742422537567",
  "segment.index.version": "v3",
  "segment.crc": "2756404785",
  "segment.realtime.status": "DONE",
  "segment.download.url": "s3://******/controller/********/****__0__1001__20250319T2215Z"
}
Could u pls help?? cc: @Mayank @Anand Kr Shaw @Xiang Fu
This is my table config
Copy code
{
  "tableName": "kyc_apps_metadata",
  "tableType": "REALTIME",
  "tenants": {
    "broker": "DefaultTenant",
    "server": "KYC",
    "tagOverrideConfig": {}
  },
  "segmentsConfig": {
    "schemaName": "kyc_apps_metadata",
    "timeColumnName": "eventEmitTime",
    "replication": "1",
    "retentionTimeUnit": null,
    "retentionTimeValue": null,
    "completionConfig": null,
    "crypterClassName": null,
    "peerSegmentDownloadScheme": null
  },
  "tableIndexConfig": {
    "nullHandlingEnabled": true,
    "loadMode": "MMAP",
    "invertedIndexColumns": [
      "platform",
      "clientType",
      "city",
      "pincode",
      "appNumber"
    ],
    "createInvertedIndexDuringSegmentGeneration": false,
    "rangeIndexColumns": [],
    "sortedColumn": [],
    "bloomFilterColumns": [],
    "bloomFilterConfigs": null,
    "noDictionaryColumns": [],
    "onHeapDictionaryColumns": [],
    "varLengthDictionaryColumns": [],
    "enableDefaultStarTree": false,
    "starTreeIndexConfigs": null,
    "enableDynamicStarTreeCreation": false,
    "segmentPartitionConfig": null,
    "columnMinMaxValueGeneratorMode": null,
    "aggregateMetrics": false,
    "streamConfigs": {
      "iamRoleBasedAccessEnabled": "true",
      "roleArn": "arn:aws:iam::*******:role/*******",
      "streamType": "kinesis",
      "stream.kinesis.topic.name": "*********",
      "region": "*********",
      "shardIteratorType": "TRIM_HORIZON",
      "stream.kinesis.consumer.type": "lowlevel",
      "stream.kinesis.fetch.timeout.millis": "30000",
      "stream.kinesis.decoder.class.name": "org.apache.pinot.plugin.stream.kafka.KafkaJSONMessageDecoder",
      "stream.kinesis.consumer.factory.class.name": "org.apache.pinot.plugin.stream.kinesis.KinesisConsumerFactory",
      "realtime.segment.flush.threshold.rows": "1000000",
      "realtime.segment.flush.threshold.time": "24h"
    },
    "DictionaryColumns": []
  },
  "metadata": {},
  "ingestionConfig": {
    "continueOnError": true,
    "filterConfig": {
      "filterFunction": "NOT STARTSWITH(eventName, 'metadata')"
    },
    "transformConfigs": [
      {
        "columnName": "ingestedAt",
        "transformFunction": "toDateTime(now(), 'yyyy-MM-dd HH:mm:ss.sss')"
      },
      {
        "columnName": "lastUpdatedAt",
        "transformFunction": "toDateTime(now(), 'yyyy-MM-dd HH:mm:ss.sss')"
      }
    ]
  },
  "quota": {
    "storage": null,
    "maxQueriesPerSecond": null
  },
  "task": null,
  "routing": {
    "instanceSelectorType": "strictReplicaGroup",
    "segmentPrunerTypes": [
      "partition"
    ]
  },
  "query": {
    "timeoutMs": null
  },
  "fieldConfigList": null,
  "upsertConfig": {
    "mode": "PARTIAL",
    "defaultPartialUpsertStrategy": "OVERWRITE",
    "partialUpsertStrategies": {
      "ingestedAt": "IGNORE"
    }
  },
  "tierConfigs": null,
  "isDimTable": false
}
My Kinesis stream has retention period as 7 days, where this 7 days data is present on shardId-000000000000, shardId-000000000001, shardId-000000000002, shardId-000000000003 shards, So yesterday, I changed the capacity mode to provisioned and provisioned 8 shards where Kinesis has created 8 new shards shardId-000000000004, shardId-000000000005, ....shardId-000000000011 But pinot is not able to consume data from the new shards... Instead after consuming data till the end of shardId-000000000000, shardId-000000000001, shardId-000000000002, shardId-000000000003 shards, it is creating thousands of empty segments...
m
@Kartik Khare ^^
x
which version of pinot are you running ?
s
Hi @Xiang Fu, its 1.2.0
x
pinot should handle all the new shards creation, can you check controller logs for
RealtimeSegmentValidationManager
?
or trigger this
RealtimeSegmentValidationManager
from the swagger ui, it’s under controller periodical task