https://pinot.apache.org/ logo
Join Slack
Powered by
``` at software.amazon.awssdk.core.internal...
# troubleshooting
s
Copy code
at software.amazon.awssdk.core.internal.http.pipeline.stages.ExecutionFailureExceptionReportingStage.execute(ExecutionFailureExceptionReportingStage.java:37)
        at software.amazon.awssdk.core.internal.http.pipeline.stages.ExecutionFailureExceptionReportingStage.execute(ExecutionFailureExceptionReportingStage.java:26)
        at software.amazon.awssdk.core.internal.http.AmazonSyncHttpClient$RequestExecutionBuilderImpl.execute(AmazonSyncHttpClient.java:193)
        at software.amazon.awssdk.core.internal.handler.BaseSyncClientHandler.invoke(BaseSyncClientHandler.java:128)
        at software.amazon.awssdk.core.internal.handler.BaseSyncClientHandler.doExecute(BaseSyncClientHandler.java:154)
        at software.amazon.awssdk.core.internal.handler.BaseSyncClientHandler.lambda$execute$1(BaseSyncClientHandler.java:107)
        at software.amazon.awssdk.core.internal.handler.BaseSyncClientHandler.measureApiCallSuccess(BaseSyncClientHandler.java:162)
        at software.amazon.awssdk.core.internal.handler.BaseSyncClientHandler.execute(BaseSyncClientHandler.java:91)
        at software.amazon.awssdk.core.client.handler.SdkSyncClientHandler.execute(SdkSyncClientHandler.java:45)
        at software.amazon.awssdk.awscore.client.handler.AwsSyncClientHandler.execute(AwsSyncClientHandler.java:55)
        at software.amazon.awssdk.services.s3.DefaultS3Client.putObject(DefaultS3Client.java:8123)
        at org.apache.pinot.plugin.filesystem.S3PinotFS.mkdir(S3PinotFS.java:305)
        ... 23 more
Caused by: java.lang.invoke.LambdaConversionException: Invalid receiver type interface org.apache.http.Header; not a subtype of implementation type interface org.apache.http.NameValuePair
        at java.lang.invoke.AbstractValidatingLambdaMetafactory.validateMetafactoryArgs(AbstractValidatingLambdaMetafactory.java:233)
        at java.lang.invoke.LambdaMetafactory.metafactory(LambdaMetafactory.java:303)
        at java.lang.invoke.CallSite.makeSite(CallSite.java:302)
        ... 69 more
java.lang.RuntimeException: Caught exception during running - org.apache.pinot.plugin.ingestion.batch.spark.SparkSegmentGenerationJobRunner
        at org.apache.pinot.spi.ingestion.batch.IngestionJobLauncher.kickoffIngestionJob(IngestionJobLauncher.java:148)
        at org.apache.pinot.spi.ingestion.batch.IngestionJobLauncher.runIngestionJob(IngestionJobLauncher.java:117)
        at org.apache.pinot.tools.admin.command.LaunchDataIngestionJobCommand.execute(LaunchDataIngestionJobCommand.java:121)
        at org.apache.pinot.tools.Command.call(Command.java:33)
        at org.apache.pinot.tools.Command.call(Command.java:29)
        at picocli.CommandLine.executeUserObject(CommandLine.java:1953)
        at picocli.CommandLine.access$1300(CommandLine.java:145)
        at picocli.CommandLine$RunLast.executeUserObjectOfLastSubcommandWithSameParent(CommandLine.java:2352)
        at picocli.CommandLine$RunLast.handle(CommandLine.java:2346)
        at picocli.CommandLine$RunLast.handle(CommandLine.java:2311)
        at picocli.CommandLine$AbstractParseResultHandler.execute(CommandLine.java:2179)
        at picocli.CommandLine.execute(CommandLine.java:2078)
        at org.apache.pinot.tools.admin.command.LaunchDataIngestionJobCommand.main(LaunchDataIngestionJobCommand.java:153)
        at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
        at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
        at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
        at java.lang.reflect.Method.invoke(Method.java:498)
        at org.apache.spark.deploy.SparkSubmit$.org$apache$spark$deploy$SparkSubmit$$runMain(SparkSubmit.scala:755)
        at org.apache.spark.deploy.SparkSubmit$.doRunMain$1(SparkSubmit.scala:180)
        at org.apache.spark.deploy.SparkSubmit$.submit(SparkSubmit.scala:205)
        at org.apache.spark.deploy.SparkSubmit$.main(SparkSubmit.scala:119)
        at org.apache.spark.deploy.SparkSubmit.main(SparkSubmit.scala)
Caused by: java.io.IOException: java.lang.BootstrapMethodError: call site initialization exception
        at org.apache.pinot.plugin.filesystem.S3PinotFS.mkdir(S3PinotFS.java:308)
        at org.apache.pinot.plugin.ingestion.batch.spark.SparkSegmentGenerationJobRunner.run(SparkSegmentGenerationJobRunner.java:145)
        at org.apache.pinot.spi.ingestion.batch.IngestionJobLauncher.kickoffIngestionJob(IngestionJobLauncher.java:146)
        ... 21 more
Caused by: java.lang.BootstrapMethodError: call site initialization exception
        at java.lang.invoke.CallSite.makeSite(CallSite.java:341)
        at java.lang.invoke.MethodHandleNatives.linkCallSiteImpl(MethodHandleNatives.java:307)
        at java.lang.invoke.MethodHandleNatives.linkCallSite(MethodHandleNatives.java:297)
        at software.amazon.awssdk.http.apache.ApacheHttpClient.transformHeaders(ApacheHttpClient.java:289)
        at software.amazon.awssdk.http.apache.ApacheHttpClient.createResponse(ApacheHttpClient.java:274)
        at software.amazon.awssdk.http.apache.ApacheHttpClient.execute(ApacheHttpClient.java:254)
        at software.amazon.awssdk.http.apache.ApacheHttpClient.access$500(ApacheHttpClient.java:106)
        at software.amazon.awssdk.http.apache.ApacheHttpClient$1.call(ApacheHttpClient.java:232)
        at software.amazon.awssdk.http.apache.ApacheHttpClient$1.call(ApacheHttpClient.java:229)
        at software.amazon.awssdk.core.internal.util.MetricUtils.measureDurationUnsafe(MetricUtils.java:64)
        at software.amazon.awssdk.core.internal.http.pipeline.stages.MakeHttpRequestStage.executeHttpRequest(MakeHttpRequestStage.java:76)
        at software.amazon.awssdk.core.internal.http.pipeline.stages.MakeHttpRequestStage.execute(MakeHttpRequestStage.java:55)
        at software.amazon.awssdk.core.internal.http.pipeline.stages.MakeHttpRequestStage.execute(MakeHttpRequestStage.java:39)
        at software.amazon.awssdk.core.internal.http.pipeline.RequestPipelineBuilder$ComposingRequestPipelineStage.execute(RequestPipelineBuilder.java:206)
        at software.amazon.awssdk.core.internal.http.pipeline.RequestPipelineBuilder$ComposingRequestPipelineStage.execute(RequestPipelineBuilder.java:206)
        at software.amazon.awssdk.core.internal.http.pipeline.RequestPipelineBuilder$ComposingRequestPipelineStage.execute(RequestPipelineBuilder.java:206)
✅ 1
k
Hi can you add the full stacktrace. Also, please try only with 0.11.0-SNAPSHOT. 0.10.0 release had issues with spark. Other than that, can you please also add your ingestion spec yaml file here
s
Copy code
executionFrameworkSpec:
  name: 'spark'
  segmentGenerationJobRunnerClassName: 'org.apache.pinot.plugin.ingestion.batch.spark.SparkSegmentGenerationJobRunner'
  segmentTarPushJobRunnerClassName: 'org.apache.pinot.plugin.ingestion.batch.spark.SparkSegmentTarPushJobRunner'
  segmentUriPushJobRunnerClassName: 'org.apache.pinot.plugin.ingestion.batch.spark.SparkSegmentUriPushJobRunner'
  segmentMetadataPushJobRunnerClassName: 'org.apache.pinot.plugin.ingestion.batch.standalone.SegmentMetadataPushJobRunner'

# Supported job types are:
#   'SegmentCreation'
#   'SegmentTarPush'
#   'SegmentUriPush'
#   'SegmentMetadataPush'
#   'SegmentCreationAndTarPush'
#   'SegmentCreationAndUriPush'
#   'SegmentCreationAndMetadataPush'
jobType: SegmentCreationAndTarPush

inputDirURI: ${inputDir}

includeFileNamePattern: 'glob:**/*.parquet'

outputDirURI: ${outputDir}

segmentCreationJobParallelism: 8

overwriteOutput: true

# pinotFSSpecs: defines all related Pinot file systems.
pinotFSSpecs:
  - # scheme: used to identify a PinotFS.
    # E.g. local, hdfs, dbfs, etc
    scheme: file
    # className: Class name used to create the PinotFS instance.
    # E.g.
    #   org.apache.pinot.spi.filesystem.LocalPinotFS is used for local filesystem
    #   org.apache.pinot.plugin.filesystem.AzurePinotFS is used for Azure Data Lake
    #   org.apache.pinot.plugin.filesystem.HadoopPinotFS is used for HDFS
    className: org.apache.pinot.spi.filesystem.LocalPinotFS
  - scheme: s3
    className: org.apache.pinot.plugin.filesystem.S3PinotFS
    configs:
      region: 'us-east-1'

# recordReaderSpec: defines all record reader
recordReaderSpec:
  dataFormat: 'parquet'
  className: 'org.apache.pinot.plugin.inputformat.parquet.ParquetRecordReader'

tableSpec:
  tableName: ${tableName}

pinotClusterSpecs:
  - # controllerURI: used to fetch table/schema information and data push.
    controllerURI:  ${controllerURI}

pushJobSpec:
  pushAttempts: 2
  pushRetryIntervalMillis: 1000

segmentNameGeneratorSpec:
  type: normalizedDate
Copy code
/spark-home/spark/bin/spark-submit \
   --class "org.apache.pinot.tools.admin.command.LaunchDataIngestionJobCommand" \
   --properties-file /spark-home/resources/conf/spark.conf \
   --files "/spark-home/pinot-resources/pinot-job-spec.yaml" \
   --jars "/spark-home/pinot-resources/plugins/pinot-batch-ingestion/pinot-batch-ingestion-standalone/pinot-batch-ingestion-standalone-0.11.0-SNAPSHOT-shaded.jar,/spark-home/pinot-resources/plugins-external/pinot-batch-ingestion/pinot-batch-ingestion-spark-2.4/pinot-batch-ingestion-spark-2.4-0.11.0-SNAPSHOT-shaded.jar,/spark-home/pinot-resources/lib/pinot-all-0.11.0-SNAPSHOT-jar-with-dependencies.jar,/spark-home/pinot-resources/plugins/pinot-file-system/pinot-s3/pinot-s3-0.11.0-SNAPSHOT-shaded.jar,/spark-home/pinot-resources/plugins/pinot-input-format/pinot-parquet/pinot-parquet-0.11.0-SNAPSHOT-shaded.jar" \
   --driver-memory "$RAM" \
   --driver-java-options "$OPTS" \
   --driver-class-path "/spark-home/pinot-resources/plugins/pinot-batch-ingestion/pinot-batch-ingestion-standalone/pinot-batch-ingestion-standalone-0.11.0-SNAPSHOT-shaded.jar,/spark-home/pinot-resources/plugins-external/pinot-batch-ingestion/pinot-batch-ingestion-spark-2.4/pinot-batch-ingestion-spark-2.4-0.11.0-SNAPSHOT-shaded.jar,/spark-home/pinot-resources/lib/pinot-all-0.11.0-SNAPSHOT-jar-with-dependencies.jar,/spark-home/pinot-resources/plugins/pinot-file-system/pinot-s3/pinot-s3-0.11.0-SNAPSHOT-shaded.jar,/spark-home/pinot-resources/plugins/pinot-input-format/pinot-parquet/pinot-parquet-0.11.0-SNAPSHOT-shaded.jar" \
   --executor-memory "$RAM" "pinot-all-0.11.0-SNAPSHOT-jar-with-dependencies.jar" \
   -jobSpecFile "/spark-home/pinot-resources/pinot-job-spec.yaml" \
   -values "inputDir=\"<s3://viglink-db-compressed-log/merchant_group/2022-07-20/>\"" \
   "outputDir=\"<s3://ad-hoc-analysis/sukesh/pinot-offline-segments/2022-07-20/>\"" \
   "tableName=\"merchant-group\"" \
   "controllerURI=\"<https://pinot.internal.qa.sovrn.startree.cloud/>\"";
k
Update: The above issue was due to older httpclient libs in spark 2.2 distribution. It got resolved after setting - 
--conf spark.driver.userClassPathFirst=true
Now there's another error due to older hadoop libs in classpath. 
mvn clean package -DuseProvidedHadoop
should fix that.
2 Views
Open in Slack
PreviousNext
Powered by