t showing flink dashboard .. below error kindly pl...
# troubleshootingy
YOGESH KUMBHARE
09/13/2023, 8:01 AMt showing flink dashboard .. below error kindly please help me asap ... as i am doing POC on that
1. org.apache.flink.client.program.ProgramInvocationException: The main method caused an error: java.util.concurrent.ExecutionException: org.apache.flink.runtime.client.JobSubmissionException: Failed to submit JobGraph.
2. at org.apache.flink.client.program.PackagedProgram.callMainMethod(PackagedProgram.java:372)
3. at org.apache.flink.client.program.PackagedProgram.invokeInteractiveModeForExecution(PackagedProgram.java:222)
4. at org.apache.flink.client.ClientUtils.executeProgram(ClientUtils.java:105)
5. at org.apache.flink.client.cli.CliFrontend.executeProgram(CliFrontend.java:851)
6. at org.apache.flink.client.cli.CliFrontend.run(CliFrontend.java:245)
7. at org.apache.flink.client.cli.CliFrontend.parseAndRun(CliFrontend.java:1095)
8. at org.apache.flink.client.cli.CliFrontend.lambda$mainInternal$9(CliFrontend.java:1189)
9. at org.apache.flink.runtime.security.contexts.NoOpSecurityContext.runSecured(NoOpSecurityContext.java:28)
10. at org.apache.flink.client.cli.CliFrontend.mainInternal(CliFrontend.java:1189)
11. at org.apache.flink.client.cli.CliFrontend.main(CliFrontend.java:1157)
12. Caused by: java.lang.RuntimeException: java.util.concurrent.ExecutionException: org.apache.flink.runtime.client.JobSubmissionException: Failed to submit JobGraph.
13. at org.apache.flink.util.ExceptionUtils.rethrow(ExceptionUtils.java:321)
14. at org.apache.flink.api.java.ExecutionEnvironment.executeAsync(ExecutionEnvironment.java:1061)
15. at org.apache.flink.client.program.ContextEnvironment.executeAsync(ContextEnvironment.java:144)
16. at org.apache.flink.client.program.ContextEnvironment.execute(ContextEnvironment.java:73)
17. at org.apache.flink.api.java.ExecutionEnvironment.execute(ExecutionEnvironment.java:942)
18. at org.apache.flink.api.java.DataSet.collect(DataSet.java:417)
19. at org.apache.flink.api.java.DataSet.print(DataSet.java:1748)
20. at org.apache.flink.examples.java.wordcount.WordCount.main(WordCount.java:96)
21. at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
22. at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
23. at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
24. at java.base/java.lang.reflect.Method.invoke(Method.java:566)
25. at org.apache.flink.client.program.PackagedProgram.callMainMethod(PackagedProgram.java:355)
26. ... 9 more
27. Caused by: java.util.concurrent.ExecutionException: org.apache.flink.runtime.client.JobSubmissionException: Failed to submit JobGraph.
28. at java.base/java.util.concurrent.CompletableFuture.reportGet(CompletableFuture.java:395)
29. at java.base/java.util.concurrent.CompletableFuture.get(CompletableFuture.java:1999)
30. at org.apache.flink.api.java.ExecutionEnvironment.executeAsync(ExecutionEnvironment.java:1056)
31. ... 20 more
32. Caused by: org.apache.flink.runtime.client.JobSubmissionException: Failed to submit JobGraph.
33. at org.apache.flink.client.program.rest.RestClusterClient.lambda$submitJob$11(RestClusterClient.java:451)
34. at java.base/java.util.concurrent.CompletableFuture.uniExceptionally(CompletableFuture.java:986)
35. at java.base/java.util.concurrent.CompletableFuture$UniExceptionally.tryFire(CompletableFuture.java:970)
36. at java.base/java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:506)
37. at java.base/java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2088)
38. at org.apache.flink.util.concurrent.FutureUtils.lambda$retryOperationWithDelay$6(FutureUtils.java:272)
39. at java.base/java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:859)
40. at java.base/java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:837)
41. at java.base/java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:506)
42. at java.base/java.util.concurrent.CompletableFuture.postFire(CompletableFuture.java:610)
43. at java.base/java.util.concurrent.CompletableFuture$UniCompose.tryFire(CompletableFuture.java:1085)
44. at java.base/java.util.concurrent.CompletableFuture$Completion.run(CompletableFuture.java:478)
45. at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1128)
46. at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628)
47. at java.base/java.lang.Thread.run(Thread.java:829)
48. Caused by: org.apache.flink.runtime.rest.util.RestClientException: [org.apache.flink.runtime.rest.handler.RestHandlerException: Could not upload job files.
49. at org.apache.flink.runtime.rest.handler.job.JobSubmitHandler.lambda$uploadJobGraphFiles$4(JobSubmitHandler.java:201)
50. at java.base/java.util.concurrent.CompletableFuture.biApply(CompletableFuture.java:1236)
51. at java.base/java.util.concurrent.CompletableFuture$BiApply.tryFire(CompletableFuture.java:1205)
52. at java.base/java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:506)
53. at java.base/java.util.concurrent.CompletableFuture$AsyncSupply.run(CompletableFuture.java:1705)
54. at java.base/java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:515)
55. at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264)
56. at java.base/java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(ScheduledThreadPoolExecutor.java:304)
57. at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1128)
58. at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628)
59. at java.base/java.lang.Thread.run(Thread.java:829)
60. Caused by: org.apache.flink.util.FlinkException: Could not upload job files.
61. at org.apache.flink.runtime.client.ClientUtils.uploadJobGraphFiles(ClientUtils.java:86)
62. at org.apache.flink.runtime.rest.handler.job.JobSubmitHandler.lambda$uploadJobGraphFiles$4(JobSubmitHandler.java:195)
63. ... 10 more
64. Caused by: java.io.IOException: Could not connect to BlobServer at address localhost/127.0.0.1:37101
65. at org.apache.flink.runtime.blob.BlobClient.<init>(BlobClient.java:103)
66. at org.apache.flink.runtime.rest.handler.job.JobSubmitHandler.lambda$null$3(JobSubmitHandler.java:199)
67. at org.apache.flink.runtime.client.ClientUtils.uploadJobGraphFiles(ClientUtils.java:82)
68. ... 11 more
69. Caused by: java.net.ConnectException: Connection refused (Connection refused)
70. at java.base/java.net.PlainSocketImpl.socketConnect(Native Method)
71. at java.base/java.net.AbstractPlainSocketImpl.doConnect(AbstractPlainSocketImpl.java:412)
72. at java.base/java.net.AbstractPlainSocketImpl.connectToAddress(AbstractPlainSocketImpl.java:255)
73. at java.base/java.net.AbstractPlainSocketImpl.connect(AbstractPlainSocketImpl.java:237)
74. at java.base/java.net.SocksSocketImpl.connect(SocksSocketImpl.java:392)
75. at java.base/java.net.Socket.connect(Socket.java:609)
76. at org.apache.flink.runtime.blob.BlobClient.<init>(BlobClient.java:97)
77. ... 13 more
78. ]
79. at org.apache.flink.runtime.rest.RestClient.parseResponse(RestClient.java:536)
80. at org.apache.flink.runtime.rest.RestClient.lambda$submitRequest$3(RestClient.java:516)
81. at java.base/java.util.concurrent.CompletableFuture$UniCompose.tryFire(CompletableFuture.java:1072)
82. ... 4 more
83. [12:59 PM] Caused by: java.io.IOException: Could not connect to BlobServer at address localhost/127.0.0.1:37101
m
Martijn Visser
09/13/2023, 8:01 AMHave you actually started the Flink cluster?
Martijn Visser
09/13/2023, 8:02 AMAre you using Docker, are you trying the Kubernetes Operator, is it a local download?
y
YOGESH KUMBHARE
09/13/2023, 8:04 AMi have just download the flink setup in local followed the ref docs
https://nightlies.apache.org/flink/flink-docs-release-1.13/docs/try-flink/local_installation/
m
Martijn Visser
09/13/2023, 8:05 AMThat's not the latest version of Flink, those instructions are at https://nightlies.apache.org/flink/flink-docs-release-1.17/docs/try-flink/local_installation/
Martijn Visser
09/13/2023, 8:05 AMSo have you ran
./bin/start-cluster.shy
YOGESH KUMBHARE
09/13/2023, 8:05 AMya correct martijn ..
m
Martijn Visser
09/13/2023, 8:05 AMWhat did it output to screen?
y
YOGESH KUMBHARE
09/13/2023, 8:07 AMactually there is two thing .. i have install flink setup in linux machine server ...
there i changed the flink-yaml file content jobmanager and taskmanager bind host ... so that i can able to see job is running on apache flink dashboard ..
YOGESH KUMBHARE
09/13/2023, 8:09 AMthe one more thing if i run wordcount program on linux machine server , not able to see on dashboard .. anu configuration need to be taken care
m
Martijn Visser
09/13/2023, 8:10 AMBut that's not what the documentation says: it first suggests to start with starting and stopping a local cluster.
Martijn Visser
09/13/2023, 8:11 AMAs also explained on https://nightlies.apache.org/flink/flink-docs-release-1.17/docs/deployment/config/
You do not need to configure any TaskManager hosts and ports, unless the setup requires the use of specific port ranges or specific network interfaces to bind to.
y
YOGESH KUMBHARE
09/13/2023, 8:15 AMi did the basic setup and ran th ecluster
YOGESH KUMBHARE
09/13/2023, 8:15 AMmy world count program jobid has been submitted .
m
Martijn Visser
09/13/2023, 8:15 AMSo after you ran the cluster, did you open the UI?
y
YOGESH KUMBHARE
09/13/2023, 8:15 AMbut i am not able to start apache flink dashboard
m
Martijn Visser
09/13/2023, 8:15 AMBefore submitting the job?
y
YOGESH KUMBHARE
09/13/2023, 8:16 AMnot able to open ui
YOGESH KUMBHARE
09/13/2023, 8:16 AMi tried with localhost:8081
YOGESH KUMBHARE
09/13/2023, 8:16 AMbut not abl eto see
m
Martijn Visser
09/13/2023, 8:16 AMThen most likely the cluster didn't start
y
YOGESH KUMBHARE
09/13/2023, 8:16 AMcluster has been started ..
m
Martijn Visser
09/13/2023, 8:16 AMYou should check the contents of the
logy
YOGESH KUMBHARE
09/13/2023, 8:17 AMone more thing .. i have install the flink in organisation dev server ..
m
Martijn Visser
09/13/2023, 8:17 AMYou're jumping from one problem to another
Martijn Visser
09/13/2023, 8:17 AMThat makes helping you really difficult
y
YOGESH KUMBHARE
09/13/2023, 8:21 AMapologies Martijn
YOGESH KUMBHARE
09/13/2023, 8:29 AMStarting taskexecutor daemon on host iaasn00297947
YOGESH KUMBHARE
09/13/2023, 8:29 AMwhen i start the cluster ..
m
Martijn Visser
09/13/2023, 8:29 AMThat's "starting", not "started" - That message should be displayed too
y
YOGESH KUMBHARE
09/13/2023, 8:29 AMbut in flink yaml file all configured in localhost
m
Martijn Visser
09/13/2023, 8:30 AM Copy code
Starting cluster.
Starting standalonesession daemon on host LR004WY402.
Starting taskexecutor daemon on host LR004WY402.y
YOGESH KUMBHARE
09/13/2023, 8:30 AMcorrect
YOGESH KUMBHARE
09/13/2023, 8:30 AMmartjin
YOGESH KUMBHARE
09/13/2023, 8:30 AMnow next what should i do ?
m
Martijn Visser
09/13/2023, 8:31 AMRun
ps aux | grep flinky
YOGESH KUMBHARE
09/13/2023, 8:32 AMcan we get into call if possible ? Martjin .. for 2 mints
YOGESH KUMBHARE
09/13/2023, 8:32 AMthat will really helped me
m
Martijn Visser
09/13/2023, 8:32 AMNo
Martijn Visser
09/13/2023, 8:32 AMI'm sorry, but that's not the way this Slack workspace works
y
YOGESH KUMBHARE
09/13/2023, 8:32 AMi can see pid for flink
m
Martijn Visser
09/13/2023, 8:32 AMThen it should work to access http://localhost:8081 if your
flink-conf.yamlMartijn Visser
09/13/2023, 8:33 AMIf not, then I can't help you, but you'll have to figure out what's blocking access to your localhost (like firewalls, policies etc)
y
YOGESH KUMBHARE
09/13/2023, 8:34 AMokay ..might be some ssl firewall .. looks like i am using dev server of organisation ..
YOGESH KUMBHARE
09/13/2023, 8:34 AMthats why its blocking the SSL ..
YOGESH KUMBHARE
09/13/2023, 8:35 AMthanks for the help Martjin
s
Sergey Nuyanzin
09/13/2023, 9:14 AMokay ..might be some ssl firewall .. looks like i am using dev server of organisationdo you have another machine like private or whatever without blockings you've mentioned it would help to double check if the issue is in firewall/organisation env or something else
y
YOGESH KUMBHARE
09/13/2023, 3:03 PMThanks @Martijn Visser
YOGESH KUMBHARE
09/13/2023, 3:05 PMNot having other Linux machine .. actually I have windows machine there Flink setup doesn’t work .. so thought to install in dev server at org level to do some poc
YOGESH KUMBHARE
09/13/2023, 3:05 PMBut not helped
YOGESH KUMBHARE
09/13/2023, 3:06 PM@Samrat Deb let me know if yiu have any imputes
s
Samrat Deb
09/14/2023, 7:03 AMPlease don't explicitly tag person. check the channel description