https://datahubproject.io logo
Join SlackCommunities
Powered by
```com.linkedin.restli.server.RestLiServiceExcepti...
# ingestion
h
Copy code
com.linkedin.restli.server.RestLiServiceException: com.datahub.util.exception.ESQueryException: Search query failed:
        at com.linkedin.metadata.restli.RestliUtil.toTask(RestliUtil.java:42)
        at com.linkedin.metadata.restli.RestliUtil.toTask(RestliUtil.java:50)
        at com.linkedin.metadata.resources.usage.UsageStats.query(UsageStats.java:320)
        at com.linkedin.metadata.resources.usage.UsageStats.queryRange(UsageStats.java:386)
        at jdk.internal.reflect.GeneratedMethodAccessor375.invoke(Unknown Source)
        at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
        at java.base/java.lang.reflect.Method.invoke(Method.java:566)
        at com.linkedin.restli.internal.server.RestLiMethodInvoker.doInvoke(RestLiMethodInvoker.java:177)
        at com.linkedin.restli.internal.server.RestLiMethodInvoker.invoke(RestLiMethodInvoker.java:333)
        at com.linkedin.restli.internal.server.filter.FilterChainDispatcherImpl.onRequestSuccess(FilterChainDispatcherImpl.java:47)
        at com.linkedin.restli.internal.server.filter.RestLiFilterChainIterator.onRequest(RestLiFilterChainIterator.java:86)
        at com.linkedin.restli.internal.server.filter.RestLiFilterChainIterator.lambda$onRequest$0(RestLiFilterChainIterator.java:73)
        at java.base/java.util.concurrent.CompletableFuture.uniAcceptNow(CompletableFuture.java:753)
        at java.base/java.util.concurrent.CompletableFuture.uniAcceptStage(CompletableFuture.java:731)
        at java.base/java.util.concurrent.CompletableFuture.thenAccept(CompletableFuture.java:2108)
        at com.linkedin.restli.internal.server.filter.RestLiFilterChainIterator.onRequest(RestLiFilterChainIterator.java:72)
        at com.linkedin.restli.internal.server.filter.RestLiFilterChain.onRequest(RestLiFilterChain.java:55)
        at com.linkedin.restli.server.BaseRestLiServer.handleResourceRequest(BaseRestLiServer.java:262)
        at com.linkedin.restli.server.RestRestLiServer.handleResourceRequestWithRestLiResponse(RestRestLiServer.java:294)
        at com.linkedin.restli.server.RestRestLiServer.handleResourceRequest(RestRestLiServer.java:262)
        at com.linkedin.restli.server.RestRestLiServer.handleResourceRequest(RestRestLiServer.java:232)
        at com.linkedin.restli.server.RestRestLiServer.doHandleRequest(RestRestLiServer.java:215)
        at com.linkedin.restli.server.RestRestLiServer.handleRequest(RestRestLiServer.java:171)
        at com.linkedin.restli.server.RestLiServer.handleRequest(RestLiServer.java:130)
        at com.linkedin.restli.server.DelegatingTransportDispatcher.handleRestRequest(DelegatingTransportDispatcher.java:70)
        at com.linkedin.r2.filter.transport.DispatcherRequestFilter.onRestRequest(DispatcherRequestFilter.java:70)
        at com.linkedin.r2.filter.TimedRestFilter.onRestRequest(TimedRestFilter.java:76)
        at com.linkedin.r2.filter.FilterChainIterator$FilterChainRestIterator.doOnRequest(FilterChainIterator.java:146)
        at com.linkedin.r2.filter.FilterChainIterator$FilterChainRestIterator.doOnRequest(FilterChainIterator.java:132)
        at com.linkedin.r2.filter.FilterChainIterator.onRequest(FilterChainIterator.java:62)
        at com.linkedin.r2.filter.TimedNextFilter.onRequest(TimedNextFilter.java:55)
        at com.linkedin.r2.filter.transport.ServerQueryTunnelFilter.onRestRequest(ServerQueryTunnelFilter.java:58)
        at com.linkedin.r2.filter.TimedRestFilter.onRestRequest(TimedRestFilter.java:76)
        at com.linkedin.r2.filter.FilterChainIterator$FilterChainRestIterator.doOnRequest(FilterChainIterator.java:146)
        at com.linkedin.r2.filter.FilterChainIterator$FilterChainRestIterator.doOnRequest(FilterChainIterator.java:132)
        at com.linkedin.r2.filter.FilterChainIterator.onRequest(FilterChainIterator.java:62)
        at com.linkedin.r2.filter.TimedNextFilter.onRequest(TimedNextFilter.java:55)
        at com.linkedin.r2.filter.message.rest.RestFilter.onRestRequest(RestFilter.java:50)
        at com.linkedin.r2.filter.TimedRestFilter.onRestRequest(TimedRestFilter.java:76)
        at com.linkedin.r2.filter.FilterChainIterator$FilterChainRestIterator.doOnRequest(FilterChainIterator.java:146)
        at com.linkedin.r2.filter.FilterChainIterator$FilterChainRestIterator.doOnRequest(FilterChainIterator.java:132)
        at com.linkedin.r2.filter.FilterChainIterator.onRequest(FilterChainIterator.java:62)
        at com.linkedin.r2.filter.FilterChainImpl.onRestRequest(FilterChainImpl.java:106)
        at com.linkedin.r2.filter.transport.FilterChainDispatcher.handleRestRequest(FilterChainDispatcher.java:75)
        at com.linkedin.r2.util.finalizer.RequestFinalizerDispatcher.handleRestRequest(RequestFinalizerDispatcher.java:61)
        at com.linkedin.r2.transport.http.server.HttpDispatcher.handleRequest(HttpDispatcher.java:101)
        at com.linkedin.r2.transport.http.server.AbstractR2Servlet.service(AbstractR2Servlet.java:105)
        at javax.servlet.http.HttpServlet.service(HttpServlet.java:790)
        at com.linkedin.restli.server.RestliHandlerServlet.service(RestliHandlerServlet.java:21)
        at com.linkedin.restli.server.RestliHandlerServlet.handleRequest(RestliHandlerServlet.java:26)
        at org.springframework.web.context.support.HttpRequestHandlerServlet.service(HttpRequestHandlerServlet.java:73)
        at javax.servlet.http.HttpServlet.service(HttpServlet.java:790)
        at org.eclipse.jetty.servlet.ServletHolder.handle(ServletHolder.java:799)
        at org.eclipse.jetty.servlet.ServletHandler$ChainEnd.doFilter(ServletHandler.java:1631)
        at com.datahub.auth.authentication.filter.AuthenticationFilter.doFilter(AuthenticationFilter.java:102)
        at org.eclipse.jetty.servlet.FilterHolder.doFilter(FilterHolder.java:193)
        at org.eclipse.jetty.servlet.ServletHandler$Chain.doFilter(ServletHandler.java:1601)
        at org.eclipse.jetty.servlet.ServletHandler.doHandle(ServletHandler.java:548)
        at org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:143)
        at org.eclipse.jetty.security.SecurityHandler.handle(SecurityHandler.java:600)
        at org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:127)
        at org.eclipse.jetty.server.handler.ScopedHandler.nextHandle(ScopedHandler.java:235)
        at org.eclipse.jetty.server.session.SessionHandler.doHandle(SessionHandler.java:1624)
        at org.eclipse.jetty.server.handler.ScopedHandler.nextHandle(ScopedHandler.java:233)
        at org.eclipse.jetty.server.handler.ContextHandler.doHandle(ContextHandler.java:1440)
        at org.eclipse.jetty.server.handler.ScopedHandler.nextScope(ScopedHandler.java:188)
        at org.eclipse.jetty.servlet.ServletHandler.doScope(ServletHandler.java:501)
        at org.eclipse.jetty.server.session.SessionHandler.doScope(SessionHandler.java:1594)
        at org.eclipse.jetty.server.handler.ScopedHandler.nextScope(ScopedHandler.java:186)
        at org.eclipse.jetty.server.handler.ContextHandler.doScope(ContextHandler.java:1355)
        at org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:141)
        at org.eclipse.jetty.server.handler.ContextHandlerCollection.handle(ContextHandlerCollection.java:191)
        at org.eclipse.jetty.server.handler.HandlerCollection.handle(HandlerCollection.java:146)
        at org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:127)
        at org.eclipse.jetty.server.Server.handle(Server.java:516)
        at org.eclipse.jetty.server.HttpChannel.lambda$handle$1(HttpChannel.java:487)
        at org.eclipse.jetty.server.HttpChannel.dispatch(HttpChannel.java:732)
        at org.eclipse.jetty.server.HttpChannel.handle(HttpChannel.java:479)
        at org.eclipse.jetty.server.HttpConnection.onFillable(HttpConnection.java:277)
        at org.eclipse.jetty.io.AbstractConnection$ReadCallback.succeeded(AbstractConnection.java:311)
        at org.eclipse.jetty.io.FillInterest.fillable(FillInterest.java:105)
        at org.eclipse.jetty.io.ChannelEndPoint$1.run(ChannelEndPoint.java:104)
        at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.runTask(EatWhatYouKill.java:338)
        at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:315)
        at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.tryProduce(EatWhatYouKill.java:173)
        at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.run(EatWhatYouKill.java:131)
        at org.eclipse.jetty.util.thread.ReservedThreadExecutor$ReservedThread.run(ReservedThreadExecutor.java:409)
        at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:883)
        at org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:1034)
        at java.base/java.lang.Thread.run(Thread.java:829)
Caused by: com.datahub.util.exception.ESQueryException: Search query failed:
        at com.linkedin.metadata.timeseries.elastic.query.ESAggregatedStatsDAO.getAggregatedStats(ESAggregatedStatsDAO.java:375)
        at com.linkedin.metadata.timeseries.elastic.ElasticSearchTimeseriesAspectService.getAggregatedStats(ElasticSearchTimeseriesAspectService.java:216)
        at com.linkedin.metadata.resources.usage.UsageStats.getBuckets(UsageStats.java:182)
        at com.linkedin.metadata.resources.usage.UsageStats.lambda$query$1(UsageStats.java:348)
        at com.linkedin.metadata.restli.RestliUtil.toTask(RestliUtil.java:30)
        ... 89 common frames omitted
Caused by: org.elasticsearch.ElasticsearchStatusException: Elasticsearch exception [type=search_phase_execution_exception, reason=all shards failed]
        at org.elasticsearch.rest.BytesRestResponse.errorFromXContent(BytesRestResponse.java:187)
        at org.elasticsearch.client.RestHighLevelClient.parseEntity(RestHighLevelClient.java:1911)
        at org.elasticsearch.client.RestHighLevelClient.parseResponseException(RestHighLevelClient.java:1888)
        at org.elasticsearch.client.RestHighLevelClient.internalPerformRequest(RestHighLevelClient.java:1645)
        at org.elasticsearch.client.RestHighLevelClient.performRequest(RestHighLevelClient.java:1602)
        at org.elasticsearch.client.RestHighLevelClient.performRequestAndParseEntity(RestHighLevelClient.java:1572)
        at org.elasticsearch.client.RestHighLevelClient.search(RestHighLevelClient.java:1088)
        at com.linkedin.metadata.timeseries.elastic.query.ESAggregatedStatsDAO.getAggregatedStats(ESAggregatedStatsDAO.java:371)
        ... 93 common frames omitted
        Suppressed: org.elasticsearch.client.ResponseException: method [POST], host
g
Hi @hundreds-airline-29192 It would be helpful for us if you post question as subject on channel then add detail in the thread of that post. Did your issue got resolved ? I can some suggestion on this thread https://datahubspace.slack.com/archives/CUMUWQU66/p1685413630484649
h
i don't understand what is the difference between first run and subsequent runs.All my services are healthy
p
Hi, @hundreds-airline-29192 I also have a similar error Can you confirm if it's similar to yours? https://datahubspace.slack.com/archives/C029A3M079U/p1685693237078179 and is the issue resolved? Thanks
h
u can try v0.10.2
p
@hundreds-airline-29192 Are you ingesting data from bigquery ? As far as I know v0.10.2 has problems with bigquery ingestion. That was resolved by v0.10.2.2 or later, so I'm using v0.10.2.2 or later by installing it. Anyway, I'll try lowering the version. thank you
h
no problems
Open in Slack
PreviousNext
Powered by