https://datahubproject.io logo
Title
e

elegant-minister-82709

05/25/2023, 6:52 PM
Hello, facing the issue when doing cluster upgrade: Logs from elasticsearch
{
  "type": "server",
  "timestamp": "2023-05-25T18:01:45,859Z",
  "level": "ERROR",
  "component": "o.e.i.g.DatabaseNodeService",
  "cluster.name": "elasticsearch",
  "node.name": "elasticsearch-master-0",
  "message": "failed to retrieve database [GeoLite2-Country.mmdb]",
  "stacktrace": [
    "org.elasticsearch.cluster.block.ClusterBlockException: blocked by: [SERVICE_UNAVAILABLE/1/state not recovered / initialized];",
    "at org.elasticsearch.cluster.block.ClusterBlocks.globalBlockedException(ClusterBlocks.java:179) ~[elasticsearch-7.17.3.jar:7.17.3]",
    "at org.elasticsearch.cluster.block.ClusterBlocks.globalBlockedRaiseException(ClusterBlocks.java:165) ~[elasticsearch-7.17.3.jar:7.17.3]",
    "at org.elasticsearch.action.search.TransportSearchAction.executeSearch(TransportSearchAction.java:929) ~[elasticsearch-7.17.3.jar:7.17.3]",
    "at org.elasticsearch.action.search.TransportSearchAction.executeLocalSearch(TransportSearchAction.java:763) ~[elasticsearch-7.17.3.jar:7.17.3]",
    "at org.elasticsearch.action.search.TransportSearchAction.lambda$executeRequest$6(TransportSearchAction.java:399) ~[elasticsearch-7.17.3.jar:7.17.3]",
    "at org.elasticsearch.action.ActionListener$1.onResponse(ActionListener.java:136) ~[elasticsearch-7.17.3.jar:7.17.3]",
    "at org.elasticsearch.index.query.Rewriteable.rewriteAndFetch(Rewriteable.java:112) ~[elasticsearch-7.17.3.jar:7.17.3]",
    "at org.elasticsearch.index.query.Rewriteable.rewriteAndFetch(Rewriteable.java:77) ~[elasticsearch-7.17.3.jar:7.17.3]",
    "at org.elasticsearch.action.search.TransportSearchAction.executeRequest(TransportSearchAction.java:487) ~[elasticsearch-7.17.3.jar:7.17.3]",
    "at org.elasticsearch.action.search.TransportSearchAction.doExecute(TransportSearchAction.java:285) ~[elasticsearch-7.17.3.jar:7.17.3]",
    "at org.elasticsearch.action.search.TransportSearchAction.doExecute(TransportSearchAction.java:101) ~[elasticsearch-7.17.3.jar:7.17.3]",
    "at org.elasticsearch.action.support.TransportAction$RequestFilterChain.proceed(TransportAction.java:179) ~[elasticsearch-7.17.3.jar:7.17.3]",
    "at org.elasticsearch.action.support.ActionFilter$Simple.apply(ActionFilter.java:53) ~[elasticsearch-7.17.3.jar:7.17.3]",
    "at org.elasticsearch.action.support.TransportAction$RequestFilterChain.proceed(TransportAction.java:177) ~[elasticsearch-7.17.3.jar:7.17.3]",
    "at org.elasticsearch.xpack.security.action.filter.SecurityActionFilter.apply(SecurityActionFilter.java:145) ~[?:?]",
    "at org.elasticsearch.action.support.TransportAction$RequestFilterChain.proceed(TransportAction.java:177) ~[elasticsearch-7.17.3.jar:7.17.3]",
    "at org.elasticsearch.action.support.TransportAction.execute(TransportAction.java:154) ~[elasticsearch-7.17.3.jar:7.17.3]",
    "at org.elasticsearch.action.support.TransportAction.execute(TransportAction.java:82) ~[elasticsearch-7.17.3.jar:7.17.3]",
    "at org.elasticsearch.client.node.NodeClient.executeLocally(NodeClient.java:95) ~[elasticsearch-7.17.3.jar:7.17.3]",
    "at org.elasticsearch.client.node.NodeClient.doExecute(NodeClient.java:73) ~[elasticsearch-7.17.3.jar:7.17.3]",
    "at org.elasticsearch.client.support.AbstractClient.execute(AbstractClient.java:407) ~[elasticsearch-7.17.3.jar:7.17.3]",
    "at org.elasticsearch.client.FilterClient.doExecute(FilterClient.java:57) ~[elasticsearch-7.17.3.jar:7.17.3]",
    "at org.elasticsearch.client.OriginSettingClient.doExecute(OriginSettingClient.java:51) ~[elasticsearch-7.17.3.jar:7.17.3]",
    "at org.elasticsearch.client.support.AbstractClient.execute(AbstractClient.java:407) ~[elasticsearch-7.17.3.jar:7.17.3]",
    "at org.elasticsearch.client.support.AbstractClient.execute(AbstractClient.java:392) ~[elasticsearch-7.17.3.jar:7.17.3]",
    "at org.elasticsearch.client.support.AbstractClient.search(AbstractClient.java:542) ~[elasticsearch-7.17.3.jar:7.17.3]",
    "at org.elasticsearch.ingest.geoip.DatabaseNodeService.lambda$retrieveDatabase$11(DatabaseNodeService.java:367) [ingest-geoip-7.17.3.jar:7.17.3]",
    "at org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingRunnable.run(ThreadContext.java:718) [elasticsearch-7.17.3.jar:7.17.3]",
    "at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) [?:?]",
    "at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) [?:?]",
    "at java.lang.Thread.run(Thread.java:833) [?:?]"
  ]
}
For anyone who is facing the same issue. We were able to fix it by simple helm redeploy. If you are using flux or argocd - just add some dummy (like
test: 123
) in the datahub helmrelease object so it will start reconcillation process and fixes himself. Looks like job datahub-nocode-migration-job doing the trick, but not sure..