https://linen.dev logo
Join Slack
Powered by
This message was deleted.
# troubleshooting
s
This message was deleted.
a
Could you please share the replication factor and other retention configs for this datasource?
m
Copy code
{
  "type": "kafka",
  "spec": {
    "dataSchema": {
      "dataSource": "analytics",
      "timestampSpec": {
        "column": "timestamp",
        "format": "millis",
        "missingValue": null
      },
      "dimensionsSpec": {
        "dimensions": [
          {
            "type": "string",
            "name": "bundle",
            "multiValueHandling": "SORTED_ARRAY",
            "createBitmapIndex": true
          }
      ],
      "granularitySpec": {
        "type": "uniform",
        "segmentGranularity": "DAY",
        "queryGranularity": "MINUTE",
        "rollup": true,
        "intervals": []
      },
      "transformSpec": {
        "filter": null,
        "transforms": []
      }
    },
    "ioConfig": {
      "topic": "analytics",
      "inputFormat": {
        "type": "json",
        "flattenSpec": {
          "useFieldDiscovery": true,
          "fields": []
        },
        "featureSpec": {},
        "keepNullColumns": true
      },
      "replicas": 1,
      "taskCount": 2,
      "taskDuration": "PT86400S",
      "consumerProperties": {
        "bootstrap.servers": "kafka:9094"
      },
      "autoScalerConfig": null,
      "pollTimeout": 100,
      "startDelay": "PT5S",
      "period": "PT30S",
      "useEarliestOffset": false,
      "completionTimeout": "PT1800S",
      "lateMessageRejectionPeriod": null,
      "earlyMessageRejectionPeriod": null,
      "lateMessageRejectionStartDateTime": null,
      "stream": "analytics",
      "useEarliestSequenceNumber": false
    },
    "tuningConfig": {
      "type": "kafka",
      "appendableIndexSpec": {
        "type": "onheap",
        "preserveExistingMetrics": false
      },
      "maxRowsInMemory": 1000000,
      "maxBytesInMemory": 0,
      "skipBytesInMemoryOverheadCheck": false,
      "maxRowsPerSegment": 5000000,
      "maxTotalRows": null,
      "intermediatePersistPeriod": "PT10M",
      "basePersistDirectory": "/opt/apache-druid-0.17.0/var/tmp/druid-realtime-persist5729124401212538543",
      "maxPendingPersists": 0,
      "indexSpec": {
        "bitmap": {
          "type": "concise"
        },
        "dimensionCompression": "lz4",
        "metricCompression": "lz4",
        "longEncoding": "longs",
        "segmentLoader": null
      },
      "indexSpecForIntermediatePersists": {
        "bitmap": {
          "type": "concise"
        },
        "dimensionCompression": "lz4",
        "metricCompression": "lz4",
        "longEncoding": "longs",
        "segmentLoader": null
      },
      "reportParseExceptions": false,
      "handoffConditionTimeout": 0,
      "resetOffsetAutomatically": false,
      "segmentWriteOutMediumFactory": null,
      "workerThreads": null,
      "chatThreads": null,
      "chatRetries": 8,
      "httpTimeout": "PT10S",
      "shutdownTimeout": "PT80S",
      "offsetFetchPeriod": "PT30S",
      "intermediateHandoffPeriod": "P2147483647D",
      "logParseExceptions": true,
      "maxParseExceptions": 2147483647,
      "maxSavedParseExceptions": 0,
      "skipSequenceNumberAvailabilityCheck": false,
      "repartitionTransitionDuration": "PT120S"
    }
  },
  "context": null,
  "suspended": false
}
a
Ah, sorry. I mean this
Does it show segments?
m
yes
a
But it doesn't seem to have much or any data
Could you please share the supervisor's and tasks' statistics and reports?
m
Copy code
{
  "dataSource": "analytics",
  "stream": "analytics",
  "partitions": 1,
  "replicas": 1,
  "durationSeconds": 86400,
  "activeTasks": [
    {
      "id": "index_kafka_analytics_9e38c45938839da_mbelbbdp",
      "startingOffsets": {
        "0": 82126
      },
      "startTime": "2022-09-14T13:40:11.133Z",
      "remainingSeconds": 12914,
      "type": "ACTIVE",
      "currentOffsets": {
        "0": 82130
      },
      "lag": {
        "0": 0
      }
    }
  ],
  "publishingTasks": [],
  "latestOffsets": {
    "0": 82130
  },
  "minimumLag": {
    "0": 0
  },
  "aggregateLag": 0,
  "offsetsLastUpdated": "2022-09-15T10:04:41.558Z",
  "suspended": false,
  "healthy": true,
  "state": "RUNNING",
  "detailedState": "RUNNING",
  "recentErrors": []
}
a
The partition offsets and the segment values match
m
Copy code
{
  "0": {
    "index_kafka_analytics_9e38c45938839da_mbelbbdp": {
      "movingAverages": {
        "buildSegments": {
          "5m": {
            "processed": 1.3668712401637382e-12,
            "unparseable": 0,
            "thrownAway": 0,
            "processedWithError": 0
          },
          "15m": {
            "processed": 0.0000019717186011055244,
            "unparseable": 0,
            "thrownAway": 0,
            "processedWithError": 0
          },
          "1m": {
            "processed": 2.0084710879147552e-51,
            "unparseable": 0,
            "thrownAway": 0,
            "processedWithError": 0
          }
        }
      },
      "totals": {
        "buildSegments": {
          "processed": 4,
          "processedWithError": 0,
          "thrownAway": 0,
          "unparseable": 0
        }
      }
    }
  }
}
a
Is this datasource not queryable?
m
is queryable!
a
So the concern is that data is missing?
m
yes . Datasource was not there until i sent and event to kafka. (but i was there yesterday)
a
Could you mark as used all segments for the datasource? Perhaps they were marked as unused?
m
Copy code
Perhaps they were marked as unused?
not by me.
a
Could you mark as used all segments for the datasource?
Does this help? Getting marked as unused is a possible explanation as to why this may have happened
m
Could you mark as used all segments for the datasource?
how can this help to the issue? i am not very sure
if dind't manually mark the segments as unused . why does the segments will get marked as unused?
a
if dind't manually mark the segments as unused . why does the segments will get marked as unused?
I'm not sure. What I can say is that all the segments becoming unused can cause the datasource to "vanish" from the UI
m
but when datasource status is unhealthy the datasource it's "vanish" from UI too. Is this a correct behaviour? or something else it is happening?
i am not so sure to flag druid as productive with this issue!!!
a
By datasource status, do you mean "supervisor status"?
m
yes! Supervisor Status
a
How old is this supervisor?
Did it have successful tasks with multiple segments published and handed off?
I guess a basic question is: Did it have multiple segments on the historicals at any point in time or is this a very new supervisor which has only had realtime segments with failing tasks?
m
i think "it is a very new supervisor which has only had realtime segments with failing tasks"
a
Thank you for the clarification. "Today I found datasource was not there" made it seem like a very old datasource with a lot of data
m
hahah! i am sorry for this misunderstanding
😄 1
ok so the issue is that task are failing before segments are published?
a
I think so, yes
Would you please be able to share why the task was failing?
m
The log is deleted as soon as task is failing!
a
druid.indexer.logs.kill.enabled
is this enabled?
m
i am not sure because i didn't specified this on the setup. But it is enabled by default i think
a
If you didn't, it shouldn't be. (It is false by default)
Have you changed the task log location from the default value?
m
even in docker?
Copy code
druid.indexer.logs.type: s3
  druid.indexer.logs.s3Bucket: dsp-druid-indexer-logs
  druid.indexer.logs.s3Prefix: druid/indexing-logs
  druid.indexer.task.restoreTasksOnRestart: "true"
a
Copy code
druid.indexer.logs.type: s3
  druid.indexer.logs.s3Bucket: dsp-druid-indexer-logs
  druid.indexer.logs.s3Prefix: druid/indexing-logs
  druid.indexer.task.restoreTasksOnRestart: "true"
Where have these been added? Could you add them in _common instead of middlemanager if that is the case?
Are logs for other tasks visible?
m
the same setup and all druid components/roles
a
I'm sorry, I didn't quite follow
Also, just to clarify, is this also a new setup? If yes, have you added and configured the s3 extension before configuring s3-based task long-term logs and possibly deep-storage?
m
this is a "new setup". I tried on the past. But i replicated this setup on a new work so it means the amazon account is new . May be there is an issue on aws s3.
a
Perhaps it is
The s3 issue may benefit from being started in a different thread (since it is a separate issue than the one in the main message)
m
sure i will check s3 setup . Deepstorage is with same s3 butcket and credentials so this can be the root .
👍 1
i will change temporary to log locallly . S3 credentials looks ok!
i am getting this error :
Copy code
2022-09-15T12:19:21,461 ERROR [task-runner-0-priority-0] org.apache.druid.indexing.seekablestream.SeekableStreamIndexTaskRunner - Encountered exception in run() before persisting. 
org.apache.kafka.common.errors.InterruptException: java.lang.InterruptedException                                                                                                   
        at org.apache.kafka.clients.consumer.internals.ConsumerNetworkClient.maybeThrowInterruptException(ConsumerNetworkClient.java:520) ~[?:?]                                    
        at org.apache.kafka.clients.consumer.internals.ConsumerNetworkClient.poll(ConsumerNetworkClient.java:281) ~[?:?]                                                            
        at org.apache.kafka.clients.consumer.internals.ConsumerNetworkClient.poll(ConsumerNetworkClient.java:236) ~[?:?]                                                            
        at org.apache.kafka.clients.consumer.KafkaConsumer.pollForFetches(KafkaConsumer.java:1297) ~[?:?]                                                                           
        at org.apache.kafka.clients.consumer.KafkaConsumer.poll(KafkaConsumer.java:1238) ~[?:?]                                                                                     
        at org.apache.kafka.clients.consumer.KafkaConsumer.poll(KafkaConsumer.java:1211) ~[?:?]                                                                                     
        at org.apache.druid.indexing.kafka.KafkaRecordSupplier.poll(KafkaRecordSupplier.java:128) ~[?:?]                                                                            
        at org.apache.druid.indexing.kafka.IncrementalPublishingKafkaIndexTaskRunner.getRecords(IncrementalPublishingKafkaIndexTaskRunner.java:95) ~[?:?]                           
        at org.apache.druid.indexing.seekablestream.SeekableStreamIndexTaskRunner.runInternal(SeekableStreamIndexTaskRunner.java:612) [druid-indexing-service-0.23.0.jar:0.23.0]    
        at org.apache.druid.indexing.seekablestream.SeekableStreamIndexTaskRunner.run(SeekableStreamIndexTaskRunner.java:265) [druid-indexing-service-0.23.0.jar:0.23.0]            
        at org.apache.druid.indexing.seekablestream.SeekableStreamIndexTask.run(SeekableStreamIndexTask.java:149) [druid-indexing-service-0.23.0.jar:0.23.0]                        
        at org.apache.druid.indexing.overlord.SingleTaskBackgroundRunner$SingleTaskBackgroundRunnerCallable.call(SingleTaskBackgroundRunner.java:477) [druid-indexing-service-0.23.0
        at org.apache.druid.indexing.overlord.SingleTaskBackgroundRunner$SingleTaskBackgroundRunnerCallable.call(SingleTaskBackgroundRunner.java:449) [druid-indexing-service-0.23.0
        at java.util.concurrent.FutureTask.run(FutureTask.java:266) [?:1.8.0_275]                                                                                                   
        at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) [?:1.8.0_275]                                                                            
        at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) [?:1.8.0_275]                                                                            
        at java.lang.Thread.run(Thread.java:748) [?:1.8.0_275]                                                                                                                      
Caused by: java.lang.InterruptedException                                                                                                                                           
        ... 17 more
a
Could we have a new thread for this as it is a separate issue, please?
m
sure
2 Views
Open in Slack
PreviousNext
Powered by