play.api.UnexpectedException: Unexpected exception[CompletionException: java.net.ConnectException: Connection refused: datahub-gms/172.18.0.5:8080]
        at play.api.http.HttpErrorHandlerExceptions$.throwableToUsefulException(HttpErrorHandler.scala:247)
        at play.api.http.DefaultHttpErrorHandler.onServerError(HttpErrorHandler.scala:176)
        at play.core.server.AkkaHttpServer$$anonfun$2.applyOrElse(AkkaHttpServer.scala:363)
        at play.core.server.AkkaHttpServer$$anonfun$2.applyOrElse(AkkaHttpServer.scala:361)
        at scala.concurrent.Future$$anonfun$recoverWith$1.apply(Future.scala:346)
        at scala.concurrent.Future$$anonfun$recoverWith$1.apply(Future.scala:345)
        at scala.concurrent.impl.CallbackRunnable.run(Promise.scala:36)
        at akka.dispatch.BatchingExecutor$AbstractBatch.processBatch(BatchingExecutor.scala:55)
        at akka.dispatch.BatchingExecutor$BlockableBatch$$anonfun$run$1.apply$mcV$sp(BatchingExecutor.scala:92)
        at akka.dispatch.BatchingExecutor$BlockableBatch$$anonfun$run$1.apply(BatchingExecutor.scala:92)
        at akka.dispatch.BatchingExecutor$BlockableBatch$$anonfun$run$1.apply(BatchingExecutor.scala:92)
        at scala.concurrent.BlockContext$.withBlockContext(BlockContext.scala:72)
        at akka.dispatch.BatchingExecutor$BlockableBatch.run(BatchingExecutor.scala:91)
        at akka.dispatch.TaskInvocation.run(AbstractDispatcher.scala:41)
        at akka.dispatch.ForkJoinExecutorConfigurator$AkkaForkJoinTask.exec(ForkJoinExecutorConfigurator.scala:49)
        at akka.dispatch.forkjoin.ForkJoinTask.doExec(ForkJoinTask.java:260)
        at akka.dispatch.forkjoin.ForkJoinPool$WorkQueue.runTask(ForkJoinPool.java:1339)
        at akka.dispatch.forkjoin.ForkJoinPool.runWorker(ForkJoinPool.java:1979)
        at akka.dispatch.forkjoin.ForkJoinWorkerThread.run(ForkJoinWorkerThread.java:107)
Caused by: java.util.concurrent.CompletionException: java.net.ConnectException: Connection refused: datahub-gms/172.18.0.5:8080
        at java.util.concurrent.CompletableFuture.encodeThrowable(CompletableFuture.java:292)
        at java.util.concurrent.CompletableFuture.completeThrowable(CompletableFuture.java:308)
        at java.util.concurrent.CompletableFuture.uniApply(CompletableFuture.java:607)
        at java.util.concurrent.CompletableFuture$UniApply.tryFire(CompletableFuture.java:591)
        at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:488)
        at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:1990)
        at scala.concurrent.java8.FuturesConvertersImpl$CF.apply(FutureConvertersImpl.scala:21)
        at scala.concurrent.java8.FuturesConvertersImpl$CF.apply(FutureConvertersImpl.scala:18)
        at scala.concurrent.impl.CallbackRunnable.run(Promise.scala:36)
        at scala.concurrent.BatchingExecutor$Batch$$anonfun$run$1.processBatch$1(BatchingExecutor.scala:63)
        at scala.concurrent.BatchingExecutor$Batch$$anonfun$run$1.apply$mcV$sp(BatchingExecutor.scala:78)
        at scala.concurrent.BatchingExecutor$Batch$$anonfun$run$1.apply(BatchingExecutor.scala:55)
        at scala.concurrent.BatchingExecutor$Batch$$anonfun$run$1.apply(BatchingExecutor.scala:55)
        at scala.concurrent.BlockContext$.withBlockContext(BlockContext.scala:72)
        at scala.concurrent.BatchingExecutor$Batch.run(BatchingExecutor.scala:54)
        at scala.concurrent.Future$InternalCallbackExecutor$.unbatchedExecute(Future.scala:601)
        at scala.concurrent.BatchingExecutor$class.execute(BatchingExecutor.scala:106)
        at scala.concurrent.Future$InternalCallbackExecutor$.execute(Future.scala:599)
        at scala.concurrent.impl.CallbackRunnable.executeWithValue(Promise.scala:44)
        at scala.concurrent.impl.Promise$DefaultPromise.tryComplete(Promise.scala:252)
        at scala.concurrent.Promise$class.complete(Promise.scala:55)
        at scala.concurrent.impl.Promise$DefaultPromise.complete(Promise.scala:157)
        at scala.concurrent.Promise$class.failure(Promise.scala:104)
        at scala.concurrent.impl.Promise$DefaultPromise.failure(Promise.scala:157)
        at play.libs.ws.ahc.StandaloneAhcWSClient$ResponseAsyncCompletionHandler.onThrowable(StandaloneAhcWSClient.java:227)
        at play.shaded.ahc.org.asynchttpclient.netty.NettyResponseFuture.abort(NettyResponseFuture.java:278)
        at play.shaded.ahc.org.asynchttpclient.netty.channel.NettyConnectListener.onFailure(NettyConnectListener.java:181)
        at play.shaded.ahc.org.asynchttpclient.netty.channel.NettyChannelConnector$1.onFailure(NettyChannelConnector.java:108)
        at play.shaded.ahc.org.asynchttpclient.netty.SimpleChannelFutureListener.operationComplete(SimpleChannelFutureListener.java:28)
        at play.shaded.ahc.org.asynchttpclient.netty.SimpleChannelFutureListener.operationComplete(SimpleChannelFutureListener.java:20)
        at play.shaded.ahc.io.netty.util.concurrent.DefaultPromise.notifyListener0(DefaultPromise.java:511)
        at play.shaded.ahc.io.netty.util.concurrent.DefaultPromise.notifyListeners0(DefaultPromise.java:504)
        at play.shaded.ahc.io.netty.util.concurrent.DefaultPromise.notifyListenersNow(DefaultPromise.java:483)
        at play.shaded.ahc.io.netty.util.concurrent.DefaultPromise.notifyListeners(DefaultPromise.java:424)
        at play.shaded.ahc.io.netty.util.concurrent.DefaultPromise.tryFailure(DefaultPromise.java:121)
        at play.shaded.ahc.io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.fulfillConnectPromise(AbstractNioChannel.java:327)
        at play.shaded.ahc.io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:343)
        at play.shaded.ahc.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:632)
        at play.shaded.ahc.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:579)
        at play.shaded.ahc.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:496)
        at play.shaded.ahc.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:458)
        at play.shaded.ahc.io.netty.util.concurrent.SingleThreadEventExecutor$5.run(SingleThreadEventExecutor.java:897)
        at play.shaded.ahc.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
        at java.lang.Thread.run(Thread.java:748)
Caused by: java.net.ConnectException: Connection refused: datahub-gms/172.18.0.5:8080
        at play.shaded.ahc.org.asynchttpclient.netty.channel.NettyConnectListener.onFailure(NettyConnectListener.java:179)
        ... 17 common frames omitted
Caused by: play.shaded.ahc.io.netty.channel.AbstractChannel$AnnotatedConnectException: Connection refused: datahub-gms/172.18.0.5:8080
        at sun.nio.ch.SocketChannelImpl.checkConnect(Native Method)
        at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:716)
        at play.shaded.ahc.io.netty.channel.socket.nio.NioSocketChannel.doFinishConnect(NioSocketChannel.java:327)
        at play.shaded.ahc.io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:340)
        ... 7 common frames omitted
Caused by: java.net.ConnectException: Connection refused
        ... 11 common frames omitted

Hi! I user ingest UI. and it's failed and throws 500 exception. then I refresh page and I'm facing this problem.

Hi there! It appears that you DataHub GMS pod is down. Do you mind sending over the logs for DataHub-GMS container?

02:44:04.113 [ForkJoinPool.commonPool-worker-15] ERROR c.l.d.g.e.DataHubDataFetcherExceptionHandler:21 - Failed to execute DataFetcher
java.util.concurrent.CompletionException: java.lang.RuntimeException: Failed to list ingestion sources
        at java.util.concurrent.CompletableFuture.encodeThrowable(CompletableFuture.java:273)
        at java.util.concurrent.CompletableFuture.completeThrowable(CompletableFuture.java:280)
        at java.util.concurrent.CompletableFuture$AsyncSupply.run(CompletableFuture.java:1606)
        at java.util.concurrent.CompletableFuture$AsyncSupply.exec(CompletableFuture.java:1596)
        at java.util.concurrent.ForkJoinTask.doExec(ForkJoinTask.java:289)
        at java.util.concurrent.ForkJoinPool$WorkQueue.runTask(ForkJoinPool.java:1056)
        at java.util.concurrent.ForkJoinPool.runWorker(ForkJoinPool.java:1692)
        at java.util.concurrent.ForkJoinWorkerThread.run(ForkJoinWorkerThread.java:175)
Caused by: java.lang.RuntimeException: Failed to list ingestion sources
        at com.linkedin.datahub.graphql.resolvers.ingest.source.ListIngestionSourcesResolver.lambda$get$0(ListIngestionSourcesResolver.java:74)
        at java.util.concurrent.CompletableFuture$AsyncSupply.run(CompletableFuture.java:1604)
        ... 5 common frames omitted
Caused by: java.lang.NullPointerException: null
        at com.linkedin.datahub.graphql.resolvers.ingest.IngestionResolverUtils.mapIngestionSource(IngestionResolverUtils.java:99)
        at com.linkedin.datahub.graphql.resolvers.ingest.IngestionResolverUtils.mapIngestionSources(IngestionResolverUtils.java:86)
        at com.linkedin.datahub.graphql.resolvers.ingest.source.ListIngestionSourcesResolver.lambda$get$0(ListIngestionSourcesResolver.java:70)
        ... 6 common frames omitted
02:44:04.114 [ForkJoinPool.commonPool-worker-6] ERROR c.datahub.graphql.GraphQLController:93 - Errors while executing graphQL query: "query listIngestionSources($input: ListIngestionSourcesInput!) {\n  listIngestionSources(input: $input) {\n    start\n    count\n    total\n    ingestionSources {\n      urn\n      name\n      type\n      config {\n        recipe\n        version\n        executorId\n        __typename\n      }\n      schedule {\n        interval\n        timezone\n        __typename\n      }\n      executions(start: 0, count: 1) {\n        start\n        count\n        total\n        executionRequests {\n          urn\n          input {\n            requestedAt\n            __typename\n          }\n          result {\n            status\n            startTimeMs\n            durationMs\n            __typename\n          }\n          __typename\n        }\n        __typename\n      }\n      __typename\n    }\n    __typename\n  }\n}\n", result: {errors=[{message=An unknown error occurred., locations=[{line=2, column=3}], path=[listIngestionSources], extensions={code=500, type=SERVER_ERROR, classification=DataFetchingException}}], data={listIngestionSources=null}}, errors: [DataHubGraphQLError{path=[listIngestionSources], code=SERVER_ERROR, locations=[SourceLocation{line=2, column=3}]}]
02:44:04.136 [pool-10-thread-1] INFO  c.l.m.filter.RestliLoggingFilter:55 - GET /entitiesV2?ids=List(urn%3Ali%3Acorpuser%3Adatahub) - batchGet - 200 - 2ms
02:44:04.148 [I/O dispatcher 1] INFO  c.l.m.k.e.ElasticsearchConnector:41 - Successfully feeded bulk request. Number of events: 1 Took time ms: -1
03:57:52.633 [pool-10-thread-1] INFO  c.l.m.filter.RestliLoggingFilter:55 - GET /entitiesV2?ids=List(urn%3Ali%3Acorpuser%3Adatahub) - batchGet - 200 - 3ms
03:57:52.658 [I/O dispatcher 1] INFO  c.l.m.k.e.ElasticsearchConnector:41 - Successfully feeded bulk request. Number of events: 1 Took time ms: -1
03:57:52.935 [pool-10-thread-1] INFO  c.l.m.filter.RestliLoggingFilter:55 - GET /entitiesV2?ids=List(urn%3Ali%3Acorpuser%3Adatahub) - batchGet - 200 - 2ms
03:57:52.949 [pool-10-thread-1] INFO  c.l.m.filter.RestliLoggingFilter:55 - GET /entitiesV2?ids=List(urn%3Ali%3Acorpuser%3Adatahub) - batchGet - 200 - 2ms
03:57:52.966 [I/O dispatcher 1] INFO  c.l.m.k.e.ElasticsearchConnector:41 - Successfully feeded bulk request. Number of events: 1 Took time ms: -1
03:57:52.971 [pool-10-thread-1] INFO  c.l.m.filter.RestliLoggingFilter:55 - GET /entitiesV2?ids=List(urn%3Ali%3Acorpuser%3Adatahub) - batchGet - 200 - 1ms
03:57:52.973 [I/O dispatcher 1] INFO  c.l.m.k.e.ElasticsearchConnector:41 - Successfully feeded bulk request. Number of events: 1 Took time ms: -1
03:57:52.977 [pool-10-thread-1] INFO  c.l.m.filter.RestliLoggingFilter:55 - GET /entitiesV2?ids=List(urn%3Ali%3Acorpuser%3Adatahub) - batchGet - 200 - 2ms
03:57:52.980 [I/O dispatcher 1] INFO  c.l.m.k.e.ElasticsearchConnector:41 - Successfully feeded bulk request. Number of events: 1 Took time ms: -1
03:57:52.986 [I/O dispatcher 1] INFO  c.l.m.k.e.ElasticsearchConnector:41 - Successfully feeded bulk request. Number of events: 1 Took time ms: -1
03:57:54.173 [ForkJoinPool.commonPool-worker-1] ERROR c.l.d.g.e.DataHubDataFetcherExceptionHandler:21 - Failed to execute DataFetcher
java.util.concurrent.CompletionException: java.lang.RuntimeException: Failed to list ingestion sources
        at java.util.concurrent.CompletableFuture.encodeThrowable(CompletableFuture.java:273)
        at java.util.concurrent.CompletableFuture.completeThrowable(CompletableFuture.java:280)
        at java.util.concurrent.CompletableFuture$AsyncSupply.run(CompletableFuture.java:1606)
        at java.util.concurrent.CompletableFuture$AsyncSupply.exec(CompletableFuture.java:1596)
        at java.util.concurrent.ForkJoinTask.doExec(ForkJoinTask.java:289)
        at java.util.concurrent.ForkJoinPool$WorkQueue.runTask(ForkJoinPool.java:1056)
        at java.util.concurrent.ForkJoinPool.runWorker(ForkJoinPool.java:1692)
        at java.util.concurrent.ForkJoinWorkerThread.run(ForkJoinWorkerThread.java:175)
Caused by: java.lang.RuntimeException: Failed to list ingestion sources
        at com.linkedin.datahub.graphql.resolvers.ingest.source.ListIngestionSourcesResolver.lambda$get$0(ListIngestionSourcesResolver.java:74)
        at java.util.concurrent.CompletableFuture$AsyncSupply.run(CompletableFuture.java:1604)
        ... 5 common frames omitted
Caused by: java.lang.NullPointerException: null
        at com.linkedin.datahub.graphql.resolvers.ingest.IngestionResolverUtils.mapIngestionSource(IngestionResolverUtils.java:99)
        at com.linkedin.datahub.graphql.resolvers.ingest.IngestionResolverUtils.mapIngestionSources(IngestionResolverUtils.java:86)
        at com.linkedin.datahub.graphql.resolvers.ingest.source.ListIngestionSourcesResolver.lambda$get$0(ListIngestionSourcesResolver.java:70)
        ... 6 common frames omitted
03:57:54.174 [ForkJoinPool.commonPool-worker-9] ERROR c.datahub.graphql.GraphQLController:93 - Errors while executing graphQL query: "query listIngestionSources($input: ListIngestionSourcesInput!) {\n  listIngestionSources(input: $input) {\n    start\n    count\n    total\n    ingestionSources {\n      urn\n      name\n      type\n      config {\n        recipe\n        version\n        executorId\n        __typename\n      }\n      schedule {\n        interval\n        timezone\n        __typename\n      }\n      executions(start: 0, count: 1) {\n        start\n        count\n        total\n        executionRequests {\n          urn\n          input {\n            requestedAt\n            __typename\n          }\n          result {\n            status\n            startTimeMs\n            durationMs\n            __typename\n          }\n          __typename\n        }\n        __typename\n      }\n      __typename\n    }\n    __typename\n  }\n}\n", result: {errors=[{message=An unknown error occurred., locations=[{line=2, column=3}], path=[listIngestionSources], extensions={code=500, type=SERVER_ERROR, classification=DataFetchingException}}], data={listIngestionSources=null}}, errors: [DataHubGraphQLError{path=[listIngestionSources], code=SERVER_ERROR, locations=[SourceLocation{line=2, column=3}]}]
03:57:54.204 [pool-10-thread-1] INFO  c.l.m.filter.RestliLoggingFilter:55 - GET /entitiesV2?ids=List(urn%3Ali%3Acorpuser%3Adatahub) - batchGet - 200 - 1ms
03:57:54.216 [I/O dispatcher 1] INFO  c.l.m.k.e.ElasticsearchConnector:41 - Successfully feeded bulk request. Number of events: 1 Took time ms: -1

thanks for reply.

Thank you @modern-zoo-97059! I think this might be a bug. Looking...

@big-carpet-38439 Yes I created from UI. Can I initialize ingestion schedule with CLI? or Do I need to access MySQL and ElasticSearch Directly?

Did you try to ingest a new schedule already? It seems that somehow the data is not aligned

@big-carpet-38439 Oh I cannot find list of ingestion source.. I try to find some hint from

datahubexecutionrequestindex_v2

in elastic search,

metadata_aspect_v2

table in mysql..and delete something.. but It doesn't change at all.. should I have to just clean all data?

@modern-zoo-97059 Do you have a lot of data already in? I don't want to make you clean if it will be hard to get back to where you are

@big-carpet-38439 Thanks! 🙂 I'll share if this problem solved!

good day have a similar problem

Hi there - What version are you on?

the version 0.8.41 / for me fix this with manual run of "datahub-datahub-restore-indices-job-template" job