https://linen.dev logo
Join Slack
Powered by
This message was deleted.
# troubleshooting
s
This message was deleted.
a
Could you please share your supervisor spec please?
j
Copy code
{
  "type": "kinesis",
  "spec": {
    "dataSchema": {
      "dataSource": "test_dev",
      "timestampSpec": {
        "column": "ts",
        "format": "millis",
        "missingValue": null
      },
      "dimensionsSpec": {
        "dimensions": [
          {
            "type": "string",
            "name": "user_id",
            "multiValueHandling": "SORTED_ARRAY",
            "createBitmapIndex": true
          },
          {
            "type": "string",
            "name": "network",
            "multiValueHandling": "SORTED_ARRAY",
            "createBitmapIndex": true
          },
          {
            "type": "string",
            "name": "proxy",
            "multiValueHandling": "SORTED_ARRAY",
            "createBitmapIndex": true
          },
          {
            "type": "string",
            "name": "status_code",
            "multiValueHandling": "SORTED_ARRAY",
            "createBitmapIndex": true
          }
        ],
        "dimensionExclusions": [
          "__time",
          "count",
          "ts"
        ],
        "includeAllDimensions": false
      },
      "metricsSpec": [
        {
          "type": "count",
          "name": "count"
        }
      ],
      "granularitySpec": {
        "type": "uniform",
        "segmentGranularity": "DAY",
        "queryGranularity": "MINUTE",
        "rollup": true,
        "intervals": []
      }
    },
    "ioConfig": {
      "stream": "test_dev",
      "inputFormat": {
        "type": "json",
        "flattenSpec": {
          "useFieldDiscovery": true,
          "fields": []
        },
        "featureSpec": {},
        "keepNullColumns": true
      },
      "endpoint": "<http://kinesis.us-east-1.amazonaws.com|kinesis.us-east-1.amazonaws.com>",
      "replicas": 1,
      "taskCount": 1,
      "taskDuration": "PT3600S",
      "startDelay": "PT5S",
      "period": "PT30S",
      "useEarliestSequenceNumber": true,
      "completionTimeout": "PT1800S",
      "lateMessageRejectionPeriod": null,
      "earlyMessageRejectionPeriod": null,
      "lateMessageRejectionStartDateTime": null,
      "recordsPerFetch": 1000,
      "fetchDelayMillis": 0,
      "awsAssumedRoleArn": null,
      "awsExternalId": null,
      "autoScalerConfig": null,
      "deaggregate": false,
      "type": "kinesis"
    },
    "tuningConfig": {
      "type": "kinesis",
      "appendableIndexSpec": {
        "type": "onheap",
        "preserveExistingMetrics": false
      },
      "maxRowsInMemory": 1000000,
      "maxBytesInMemory": 0,
      "skipBytesInMemoryOverheadCheck": false,
      "maxRowsPerSegment": 5000000,
      "maxTotalRows": null,
      "intermediatePersistPeriod": "PT10M",
      "basePersistDirectory": "/opt/druid/var/tmp/druid-realtime-persist7423828671201627472",
      "maxPendingPersists": 0,
      "indexSpec": {
        "bitmap": {
          "type": "roaring",
          "compressRunOnSerialization": true
        },
        "dimensionCompression": "lz4",
        "metricCompression": "lz4",
        "longEncoding": "longs",
        "segmentLoader": null
      },
      "indexSpecForIntermediatePersists": {
        "bitmap": {
          "type": "roaring",
          "compressRunOnSerialization": true
        },
        "dimensionCompression": "lz4",
        "metricCompression": "lz4",
        "longEncoding": "longs",
        "segmentLoader": null
      },
      "reportParseExceptions": false,
      "handoffConditionTimeout": 0,
      "resetOffsetAutomatically": false,
      "skipSequenceNumberAvailabilityCheck": true,
      "segmentWriteOutMediumFactory": null,
      "workerThreads": null,
      "chatThreads": null,
      "chatRetries": 8,
      "httpTimeout": "PT10S",
      "shutdownTimeout": "PT80S",
      "recordBufferSize": 10000,
      "recordBufferOfferTimeout": 5000,
      "recordBufferFullWait": 5000,
      "fetchThreads": null,
      "logParseExceptions": false,
      "maxParseExceptions": 2147483647,
      "maxSavedParseExceptions": 0,
      "maxRecordsPerPoll": 100,
      "intermediateHandoffPeriod": "P2147483647D",
      "repartitionTransitionDuration": "PT120S",
      "offsetFetchPeriod": "PT30S",
      "useListShards": true,
      "skipIgnorableShards": false
    }
  },
  "context": null
}
a
Also, is the stream being read by other consumers? https://docs.aws.amazon.com/streams/latest/dev/service-sizes-and-limits.html may help find the limits and increase them if necessary
j
yeah it is
a
I think the limit of 5TPS per shard is being hit
j
nothing I could change at spec level to avoid it?
a
Is it affecting ingestion? I see that this log is from the supervisor
j
yeah basically no ingestion at all
a
Is it also affecting the other consumer (Ingestion?)
There is one solution if the tasks are polling too frequently
j
I only have 1 ingestion in that cluster consuming from that stream, and other consumer in other druid cluster
a
fetchDelayMillis
can be set to 100 or 200ms
But this would help with tasks
I'm not sure it would with the supervisor. Could you please share the entire stacktrace corresponding to the WARN log?
j
also, in the tasks I get the following log
task
but the serviceaccount has a role with a policy with required permissions, otherwise I would not see the preliminary results when setting up the ingestion spec
wondering, the serviceaccount is applied to each deployment (broker, coordinator, router), what service is in charge of indexing/indexing data? The kinesis permission issue seems to be only coming from the Tasks being created so I think there is some service which does not have the serviceaccount applied somehow
a
Have you applied it to the middlemanagers (I think peons inherit this property) or Indexers if you're using them?
j
thats what I am wondering .. I applied the serviceaccount to deployments this way in pulumi
Copy code
transformations: [(obj, opts): void => {
                // Link service account to IAM role we created
                if (obj.kind === 'Deployment') {
                    obj.spec.template.spec.serviceAccountName = serviceAccountName;
                }
            }],
that result in following deployments having the serviceaccount:
Copy code
druid-cluster-test-broker                                                                                                                   
druid-cluster-test-coordinator                                                                          
druid-cluster-test-router
they are the only deployments
s
middle managers and historicals use statefulsets in the helm chart, can you apply service account to those as well?
j
hmm interesting, gotcha, applying them too
it worked!
🎉 1
however I am getting new issue, getting rate limited by AWS
Copy code
2022-09-21T18:51:09,629 WARN [KinesisRecordSupplier-Worker-1] org.apache.druid.indexing.kinesis.KinesisRecordSupplier - encounted ProvisionedThroughputExceededException while fetching records, this means that the request rate for the stream is too high, or the requested data is too large for the available throughput. Reduce the frequency or size of your requests.
com.amazonaws.services.kinesis.model.ProvisionedThroughputExceededException: Rate exceeded for Shard - 123756343567/test_dev/shardId-000000000014 (Service: AmazonKinesis; Status Code: 400; Error Code: ProvisionedThroughputExceededException; Request ID: c1db5b18-274a-13db-9a7e-0445eb3dab18; Proxy: null)
	at com.amazonaws.http.AmazonHttpClient$RequestExecutor.handleErrorResponse(AmazonHttpClient.java:1879) ~[aws-java-sdk-core-1.12.264.jar:?]
	at com.amazonaws.http.AmazonHttpClient$RequestExecutor.handleServiceErrorResponse(AmazonHttpClient.java:1418) ~[aws-java-sdk-core-1.12.264.jar:?]
	at com.amazonaws.http.AmazonHttpClient$RequestExecutor.executeOneRequest(AmazonHttpClient.java:1387) ~[aws-java-sdk-core-1.12.264.jar:?]
	at com.amazonaws.http.AmazonHttpClient$RequestExecutor.executeHelper(AmazonHttpClient.java:1157) ~[aws-java-sdk-core-1.12.264.jar:?]
	at com.amazonaws.http.AmazonHttpClient$RequestExecutor.doExecute(AmazonHttpClient.java:814) ~[aws-java-sdk-core-1.12.264.jar:?]
	at com.amazonaws.http.AmazonHttpClient$RequestExecutor.executeWithTimer(AmazonHttpClient.java:781) ~[aws-java-sdk-core-1.12.264.jar:?]
	at com.amazonaws.http.AmazonHttpClient$RequestExecutor.execute(AmazonHttpClient.java:755) ~[aws-java-sdk-core-1.12.264.jar:?]
	at com.amazonaws.http.AmazonHttpClient$RequestExecutor.access$500(AmazonHttpClient.java:715) ~[aws-java-sdk-core-1.12.264.jar:?]
	at com.amazonaws.http.AmazonHttpClient$RequestExecutionBuilderImpl.execute(AmazonHttpClient.java:697) ~[aws-java-sdk-core-1.12.264.jar:?]
	at com.amazonaws.http.AmazonHttpClient.execute(AmazonHttpClient.java:561) ~[aws-java-sdk-core-1.12.264.jar:?]
	at com.amazonaws.http.AmazonHttpClient.execute(AmazonHttpClient.java:541) ~[aws-java-sdk-core-1.12.264.jar:?]
	at com.amazonaws.services.kinesis.AmazonKinesisClient.doInvoke(AmazonKinesisClient.java:2980) ~[?:?]
	at com.amazonaws.services.kinesis.AmazonKinesisClient.invoke(AmazonKinesisClient.java:2947) ~[?:?]
	at com.amazonaws.services.kinesis.AmazonKinesisClient.invoke(AmazonKinesisClient.java:2936) ~[?:?]
	at com.amazonaws.services.kinesis.AmazonKinesisClient.executeGetRecords(AmazonKinesisClient.java:1330) ~[?:?]
	at com.amazonaws.services.kinesis.AmazonKinesisClient.getRecords(AmazonKinesisClient.java:1299) ~[?:?]
	at org.apache.druid.indexing.kinesis.KinesisRecordSupplier$PartitionResource.lambda$fetchRecords$1(KinesisRecordSupplier.java:240) ~[?:?]
	at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:511) ~[?:1.8.0_275]
	at java.util.concurrent.FutureTask.run(FutureTask.java:266) ~[?:1.8.0_275]
	at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.access$201(ScheduledThreadPoolExecutor.java:180) ~[?:1.8.0_275]
	at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(ScheduledThreadPoolExecutor.java:293) ~[?:1.8.0_275]
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_275]
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_275]
	at java.lang.Thread.run(Thread.java:748) ~[?:1.8.0_275]
a
j
added it but no effect yet
"fetchDelayMillis": 200,
s
How big are the messages, you are retrieving 1000 msg per request based on 
"recordsPerFetch": 1000
but maybe you are hitting the 10MB limit on the fetch?
j
They should not be that big I think
I think it's the fact that there is another consumer fetching data from same stream
👀 1
s
so if the limit is 5TPS, 
"fetchDelayMillis": 200
puts you right at the limit for a single ingestion, if you have two of them, maybe that's the issue. If you need to have both running, I think you need to set that to 
400
on both of them?
j
yeah will try that
Open in Slack
PreviousNext
Powered by