Job manager and task manager pods are not coming u...
# troubleshooting
j
Job manager and task manager pods are not coming up with below error, it was working fine till yesterday. How do we fix?
Copy code
2023-10-11 08:15:00,891 WARN  io.fabric8.kubernetes.client.Config                          [] - Error reading service account token from: [/var/run/secrets/kubernetes.io/serviceaccount/token]. Ignoring.
82
2023-10-11 08:15:00,957 INFO  org.apache.flink.runtime.entrypoint.ClusterEntrypoint        [] - Shutting KubernetesSessionClusterEntrypoint down with application status FAILED. Diagnostics org.apache.flink.util.FlinkException: Could not create the ha services from the instantiated HighAvailabilityServicesFactory org.apache.flink.kubernetes.highavailability.KubernetesHaServicesFactory.
81
	at org.apache.flink.runtime.highavailability.HighAvailabilityServicesUtils.createCustomHAServices(HighAvailabilityServicesUtils.java:299)
80
	at org.apache.flink.runtime.highavailability.HighAvailabilityServicesUtils.createCustomHAServices(HighAvailabilityServicesUtils.java:285)
79
	at org.apache.flink.runtime.highavailability.HighAvailabilityServicesUtils.createHighAvailabilityServices(HighAvailabilityServicesUtils.java:145)
78
	at org.apache.flink.runtime.entrypoint.ClusterEntrypoint.createHaServices(ClusterEntrypoint.java:439)
77
	at org.apache.flink.runtime.entrypoint.ClusterEntrypoint.initializeServices(ClusterEntrypoint.java:382)
76
	at org.apache.flink.runtime.entrypoint.ClusterEntrypoint.runCluster(ClusterEntrypoint.java:282)
75
	at org.apache.flink.runtime.entrypoint.ClusterEntrypoint.lambda$startCluster$1(ClusterEntrypoint.java:232)
74
	at org.apache.flink.runtime.security.contexts.NoOpSecurityContext.runSecured(NoOpSecurityContext.java:28)
73
	at org.apache.flink.runtime.entrypoint.ClusterEntrypoint.startCluster(ClusterEntrypoint.java:229)
72
	at org.apache.flink.runtime.entrypoint.ClusterEntrypoint.runClusterEntrypoint(ClusterEntrypoint.java:729)
71
	at org.apache.flink.kubernetes.entrypoint.KubernetesSessionClusterEntrypoint.main(KubernetesSessionClusterEntrypoint.java:61)
70
Caused by: io.fabric8.kubernetes.client.KubernetesClientException: Failure executing: GET at: <https://x.x.x.x/api/v1/namespaces/flink/configmaps?labelSelector=app%3Dflink-cluster%2Ctype%3Dflink-native-kubernetes%2Cconfigmap-type%3Dhigh-availability>. Message: Unauthorized. Received status: Status(apiVersion=v1, code=401, details=null, kind=Status, message=Unauthorized, metadata=ListMeta(_continue=null, remainingItemCount=null, resourceVersion=null, selfLink=null, additionalProperties={}), reason=Unauthorized, status=Failure, additionalProperties={}).
69
	at io.fabric8.kubernetes.client.dsl.base.OperationSupport.requestFailure(OperationSupport.java:684)
68
	at io.fabric8.kubernetes.client.dsl.base.OperationSupport.requestFailure(OperationSupport.java:664)
67
	at io.fabric8.kubernetes.client.dsl.base.OperationSupport.assertResponseCode(OperationSupport.java:615)
66
	at io.fabric8.kubernetes.client.dsl.base.OperationSupport.handleResponse(OperationSupport.java:558)
65
	at io.fabric8.kubernetes.client.dsl.base.OperationSupport.handleResponse(OperationSupport.java:521)
64
	at io.fabric8.kubernetes.client.dsl.base.OperationSupport.handleResponse(OperationSupport.java:505)
63
	at io.fabric8.kubernetes.client.dsl.base.BaseOperation.listRequestHelper(BaseOperation.java:133)
62
	at io.fabric8.kubernetes.client.dsl.base.BaseOperation.list(BaseOperation.java:415)
61
	at io.fabric8.kubernetes.client.dsl.base.BaseOperation.list(BaseOperation.java:83)
60
	at io.fabric8.kubernetes.client.informers.cache.Reflector.listSyncAndWatch(Reflector.java:81)
59
	at io.fabric8.kubernetes.client.informers.impl.DefaultSharedIndexInformer.run(DefaultSharedIndexInformer.java:146)
58
	at io.fabric8.kubernetes.client.dsl.base.BaseOperation.inform(BaseOperation.java:961)
57
	at org.apache.flink.kubernetes.kubeclient.resources.KubernetesSharedInformer.<init>(KubernetesSharedInformer.java:78)
56
	at org.apache.flink.kubernetes.kubeclient.resources.KubernetesConfigMapSharedInformer.<init>(KubernetesConfigMapSharedInformer.java:38)
55
	at org.apache.flink.kubernetes.kubeclient.Fabric8FlinkKubeClient.createConfigMapSharedWatcher(Fabric8FlinkKubeClient.java:356)
54
	at org.apache.flink.kubernetes.highavailability.KubernetesMultipleComponentLeaderElectionHaServices.<init>(KubernetesMultipleComponentLeaderElectionHaServices.java:98)
53
	at org.apache.flink.kubernetes.highavailability.KubernetesHaServicesFactory.createHAServices(KubernetesHaServicesFactory.java:41)
52
	at org.apache.flink.runtime.highavailability.HighAvailabilityServicesUtils.createCustomHAServices(HighAvailabilityServicesUtils.java:296)
51
	... 10 more