This message was deleted.
# troubleshootings
Slackbot
03/20/2023, 1:20 PMThis message was deleted.
m
Martin Maqueira
03/20/2023, 1:29 PMThe task setup is it this :
Martin Maqueira
03/20/2023, 1:30 PM Copy code
{
"type": "kafka",
"spec": {
"dataSchema": {
"dataSource": "qps",
"timestampSpec": {
"column": "timestamp",
"format": "millis",
"missingValue": null
},
"dimensionsSpec": {
"dimensions": [
{
"type": "string",
"name": "bidderId",
"multiValueHandling": "SORTED_ARRAY",
"createBitmapIndex": true
},
{
"type": "string",
"name": "exchange",
"multiValueHandling": "SORTED_ARRAY",
"createBitmapIndex": true
}
],
"dimensionExclusions": [
"qps",
"count",
"timestamp"
]
},
"metricsSpec": [
{
"type": "longSum",
"name": "qps",
"fieldName": "qps",
"expression": null
},
{
"type": "count",
"name": "count"
}
],
"granularitySpec": {
"type": "uniform",
"segmentGranularity": "MONTH",
"queryGranularity": "MINUTE",
"rollup": true,
"intervals": null
},
"transformSpec": {
"filter": null,
"transforms": [
{
"type": "expression",
"name": "qps",
"expression": "\"qps\" / 60"
}
]
}
},
"ioConfig": {
"topic": "qps",
"inputFormat": {
"type": "json",
"flattenSpec": {
"useFieldDiscovery": true,
"fields": []
},
"featureSpec": {}
},
"replicas": 1,
"taskCount": 1,
"taskDuration": "PT10M",
"consumerProperties": {
"bootstrap.servers": "kafka-0:9094,kafka-1:9094,kafka-2:9094"
},
"pollTimeout": 5000,
"startDelay": "PT5S",
"period": "PT30S",
"useEarliestOffset": false,
"completionTimeout": "PT5M",
"lateMessageRejectionPeriod": null,
"earlyMessageRejectionPeriod": null,
"lateMessageRejectionStartDateTime": null,
"stream": "qps",
"useEarliestSequenceNumber": false,
"type": "kafka"
},
"tuningConfig": {
"type": "kafka",
"maxRowsInMemory": 1000000,
"maxBytesInMemory": 0,
"maxRowsPerSegment": 5000000,
"maxTotalRows": null,
"intermediatePersistPeriod": "PT1M",
"basePersistDirectory": "/opt/apache-druid-0.17.0/var/tmp/druid-realtime-persist3217785953270375002",
"maxPendingPersists": 0,
"indexSpec": {
"bitmap": {
"type": "concise"
},
"dimensionCompression": "lz4",
"metricCompression": "lz4",
"longEncoding": "longs",
"segmentLoader": null
},
"indexSpecForIntermediatePersists": {
"bitmap": {
"type": "concise"
},
"dimensionCompression": "lz4",
"metricCompression": "lz4",
"longEncoding": "longs",
"segmentLoader": null
},
"buildV9Directly": true,
"reportParseExceptions": false,
"handoffConditionTimeout": 0,
"resetOffsetAutomatically": false,
"segmentWriteOutMediumFactory": null,
"workerThreads": null,
"chatThreads": null,
"chatRetries": 8,
"httpTimeout": "PT10S",
"shutdownTimeout": "PT80S",
"offsetFetchPeriod": "PT30S",
"intermediateHandoffPeriod": "P2147483647D",
"logParseExceptions": true,
"maxParseExceptions": 2147483647,
"maxSavedParseExceptions": 0,
"skipSequenceNumberAvailabilityCheck": false,
"repartitionTransitionDuration": "PT120S"
}
}
}Martin Maqueira
03/20/2023, 1:31 PMsome values all toghether for easy reading:
n
Nick M
03/20/2023, 2:00 PMThis is the configured behaviour. The task will run for <taskDuration>, and then wait for <completionTimeout> to handoff segments. Once the completion timeout is reached, the supervisor will terminate the task with the error message above...
Your coordinator is responsible for handoffs so you may need to poke around the duty metrics of the coordinator to see what's taking so long
👍 1
m
Martin Maqueira
03/20/2023, 2:06 PM@Nick M.. to be sure i get the timing , Total time it will at list the taskDuration. If time it's not enoguh the process will still doing maximum until CompletionTimeout. is That right? So the max time will be taskDuration+CompletionTimeout. Yes?
n
Nick M
03/20/2023, 2:08 PMYes...in a nutshell...
taskduration = reading/ingesting
completionTimeout = merging/handing off
totaltime = taskduration+completiontimeout
m
Martin Maqueira
03/21/2023, 4:10 PMThanks @Nick M The issue was because some segments where unavailable an Historical was trying load them. The issue was solved when i deleted the this segments. They where more than 200 of unavailable segments.