➜  ~ kubectl describe pod prerequisites-mysql-0
Name:         prerequisites-mysql-0
Namespace:    prod-it-data
Priority:     0
Node:         <http://kube00560.taboolasyndication.com/10.110.108.239|kube00560.taboolasyndication.com/10.110.108.239>
Start Time:   Thu, 01 Sep 2022 09:02:29 +0300
Labels:       <http://app.kubernetes.io/component=primary|app.kubernetes.io/component=primary>
              <http://app.kubernetes.io/instance=prerequisites|app.kubernetes.io/instance=prerequisites>
              <http://app.kubernetes.io/managed-by=Helm|app.kubernetes.io/managed-by=Helm>
              <http://app.kubernetes.io/name=mysql|app.kubernetes.io/name=mysql>
              controller-revision-hash=prerequisites-mysql-765bc74f8c
              <http://helm.sh/chart=mysql-9.1.2|helm.sh/chart=mysql-9.1.2>
              <http://statefulset.kubernetes.io/pod-name=prerequisites-mysql-0|statefulset.kubernetes.io/pod-name=prerequisites-mysql-0>
Annotations:  checksum/configuration: f25419cba89da9112afaebb339431970fb516b7719d448d12799914417968846
              <http://kubernetes.io/limit-ranger|kubernetes.io/limit-ranger>: LimitRanger plugin set: cpu, memory request for container mysql; cpu, memory limit for container mysql
              <http://podpreset.admission.kubernetes.io/podpreset-cluster-domain|podpreset.admission.kubernetes.io/podpreset-cluster-domain>: 6821103621
Status:       Running
IP:           10.134.102.189
IPs:
  IP:           10.134.102.189
Controlled By:  StatefulSet/prerequisites-mysql
Containers:
  mysql:
    Container ID:   <containerd://72d37d9195e176ca0be7f467b7f45ea330857c981d0d71edeb124a9494238f6>2
    Image:          <http://docker.io/bitnami/mysql:8.0.29-debian-10-r23|docker.io/bitnami/mysql:8.0.29-debian-10-r23>
    Image ID:       <http://docker.io/bitnami/mysql@sha256:a4c097f505825077d7983c740dcf099f8b40ec7d318aecfd9891337600145a2a|docker.io/bitnami/mysql@sha256:a4c097f505825077d7983c740dcf099f8b40ec7d318aecfd9891337600145a2a>
    Port:           3306/TCP
    Host Port:      0/TCP
    State:          Running
      Started:      Thu, 01 Sep 2022 09:02:57 +0300
    Ready:          False
    Restart Count:  0
    Limits:
      cpu:     1
      memory:  512Mi
    Requests:
      cpu:     100m
      memory:  256Mi
    Liveness:  exec [/bin/bash -ec password_aux="${MYSQL_ROOT_PASSWORD:-}"
if [[ -f "${MYSQL_ROOT_PASSWORD_FILE:-}" ]]; then
    password_aux=$(cat "$MYSQL_ROOT_PASSWORD_FILE")
fi
mysqladmin status -uroot -p"${password_aux}"
] delay=5s timeout=1s period=10s #success=1 #failure=3
    Readiness:  exec [/bin/bash -ec password_aux="${MYSQL_ROOT_PASSWORD:-}"
if [[ -f "${MYSQL_ROOT_PASSWORD_FILE:-}" ]]; then
    password_aux=$(cat "$MYSQL_ROOT_PASSWORD_FILE")
fi
mysqladmin status -uroot -p"${password_aux}"
] delay=5s timeout=1s period=10s #success=1 #failure=3
    Startup:  exec [/bin/bash -ec password_aux="${MYSQL_ROOT_PASSWORD:-}"
if [[ -f "${MYSQL_ROOT_PASSWORD_FILE:-}" ]]; then
    password_aux=$(cat "$MYSQL_ROOT_PASSWORD_FILE")
fi
mysqladmin status -uroot -p"${password_aux}"
] delay=15s timeout=1s period=10s #success=1 #failure=10
    Environment:
      BITNAMI_DEBUG:        false
      MYSQL_ROOT_PASSWORD:  <set to the key 'mysql-root-password' in secret 'mysql-secrets'>  Optional: false
      MYSQL_DATABASE:       my_database
      cluster_domain:       <http://taboolasyndication.com|taboolasyndication.com>
    Mounts:
      /bitnami/mysql from data (rw)
      /opt/bitnami/mysql/conf/my.cnf from config (rw,path="my.cnf")
      /var/run/secrets/kubernetes.io/serviceaccount from prerequisites-mysql-token-dthdp (ro)
Conditions:
  Type              Status
  Initialized       True
  Ready             False
  ContainersReady   False
  PodScheduled      True
Volumes:
  data:
    Type:       PersistentVolumeClaim (a reference to a PersistentVolumeClaim in the same namespace)
    ClaimName:  data-prerequisites-mysql-0
    ReadOnly:   false
  config:
    Type:      ConfigMap (a volume populated by a ConfigMap)
    Name:      prerequisites-mysql
    Optional:  false
  prerequisites-mysql-token-dthdp:
    Type:        Secret (a volume populated by a Secret)
    SecretName:  prerequisites-mysql-token-dthdp
    Optional:    false
QoS Class:       Burstable
Node-Selectors:  <none>
Tolerations:     <http://node.kubernetes.io/not-ready:NoExecute|node.kubernetes.io/not-ready:NoExecute> op=Exists for 300s
                 <http://node.kubernetes.io/unreachable:NoExecute|node.kubernetes.io/unreachable:NoExecute> op=Exists for 300s
Events:
  Type     Reason            Age               From               Message
  ----     ------            ----              ----               -------
  Warning  FailedScheduling  87s               default-scheduler  0/854 nodes are available: 854 pod has unbound immediate PersistentVolumeClaims.
  Warning  FailedScheduling  87s               default-scheduler  0/854 nodes are available: 854 pod has unbound immediate PersistentVolumeClaims.
  Normal   Scheduled         85s               default-scheduler  Successfully assigned prod-it-data/prerequisites-mysql-0 to <http://kube00560.taboolasyndication.com|kube00560.taboolasyndication.com>
  Normal   Pulling           79s               kubelet            Pulling image "<http://docker.io/bitnami/mysql:8.0.29-debian-10-r23|docker.io/bitnami/mysql:8.0.29-debian-10-r23>"
  Normal   Pulled            60s               kubelet            Successfully pulled image "<http://docker.io/bitnami/mysql:8.0.29-debian-10-r23|docker.io/bitnami/mysql:8.0.29-debian-10-r23>" in 19.14969436s
  Normal   Created           57s               kubelet            Created container mysql
  Normal   Started           57s               kubelet            Started container mysql
  Warning  Unhealthy         1s (x5 over 41s)  kubelet            Startup probe failed: mysqladmin: [Warning] Using a password on the command line interface can be insecure.
mysqladmin: connect to server at 'localhost' failed
error: 'Can't connect to local MySQL server through socket '/opt/bitnami/mysql/tmp/mysql.sock' (2)'
Check that mysqld is running and that the socket: '/opt/bitnami/mysql/tmp/mysql.sock' exists!

Caused by: java.net.ConnectException: Timeout connecting to [elasticsearch-master/10.136.13.77:9200]

~ kubectl logs datahub-kafka-setup-job-clttk
[main] INFO org.apache.kafka.clients.admin.AdminClientConfig - AdminClientConfig values:
	bootstrap.servers = [prerequisites-kafka:9092]
	client.dns.lookup = use_all_dns_ips
	client.id =
	<http://connections.max.idle.ms|connections.max.idle.ms> = 300000
	<http://default.api.timeout.ms|default.api.timeout.ms> = 60000
	<http://metadata.max.age.ms|metadata.max.age.ms> = 300000
	metric.reporters = []
	metrics.num.samples = 2
	metrics.recording.level = INFO
	<http://metrics.sample.window.ms|metrics.sample.window.ms> = 30000
	receive.buffer.bytes = 65536
	<http://reconnect.backoff.max.ms|reconnect.backoff.max.ms> = 1000
	<http://reconnect.backoff.ms|reconnect.backoff.ms> = 50
	<http://request.timeout.ms|request.timeout.ms> = 30000
	retries = 2147483647
	<http://retry.backoff.ms|retry.backoff.ms> = 100
	sasl.client.callback.handler.class = null
	sasl.jaas.config = null
	sasl.kerberos.kinit.cmd = /usr/bin/kinit
	sasl.kerberos.min.time.before.relogin = 60000
	sasl.kerberos.service.name = null
	sasl.kerberos.ticket.renew.jitter = 0.05
	sasl.kerberos.ticket.renew.window.factor = 0.8
	sasl.login.callback.handler.class = null
	sasl.login.class = null
	sasl.login.refresh.buffer.seconds = 300
	sasl.login.refresh.min.period.seconds = 60
	sasl.login.refresh.window.factor = 0.8
	sasl.login.refresh.window.jitter = 0.05
	sasl.mechanism = GSSAPI
	security.protocol = PLAINTEXT
	security.providers = null
	send.buffer.bytes = 131072
	<http://socket.connection.setup.timeout.max.ms|socket.connection.setup.timeout.max.ms> = 127000
	<http://socket.connection.setup.timeout.ms|socket.connection.setup.timeout.ms> = 10000
	ssl.cipher.suites = null
	ssl.enabled.protocols = [TLSv1.2]
	ssl.endpoint.identification.algorithm = https
	ssl.engine.factory.class = null
	ssl.key.password = null
	ssl.keymanager.algorithm = SunX509
	ssl.keystore.certificate.chain = null
	ssl.keystore.key = null
	ssl.keystore.location = null
	ssl.keystore.password = null
	ssl.keystore.type = JKS
	ssl.protocol = TLSv1.2
	ssl.provider = null
	ssl.secure.random.implementation = null
	ssl.trustmanager.algorithm = PKIX
	ssl.truststore.certificates = null
	ssl.truststore.location = null
	ssl.truststore.password = null
	ssl.truststore.type = JKS

[main] INFO org.apache.kafka.common.utils.AppInfoParser - Kafka version: 6.1.4-ccs
[main] INFO org.apache.kafka.common.utils.AppInfoParser - Kafka commitId: c9124241a6ff43bc
[main] INFO org.apache.kafka.common.utils.AppInfoParser - Kafka startTimeMs: 1662018258463
WARNING: Due to limitations in metric names, topics with a period ('.') or underscore ('_') could collide. To avoid issues it is best to use either, but not both.
WARNING: Due to limitations in metric names, topics with a period ('.') or underscore ('_') could collide. To avoid issues it is best to use either, but not both.
WARNING: Due to limitations in metric names, topics with a period ('.') or underscore ('_') could collide. To avoid issues it is best to use either, but not both.
WARNING: Due to limitations in metric names, topics with a period ('.') or underscore ('_') could collide. To avoid issues it is best to use either, but not both.
WARNING: Due to limitations in metric names, topics with a period ('.') or underscore ('_') could collide. To avoid issues it is best to use either, but not both.
WARNING: Due to limitations in metric names, topics with a period ('.') or underscore ('_') could collide. To avoid issues it is best to use either, but not both.
WARNING: Due to limitations in metric names, topics with a period ('.') or underscore ('_') could collide. To avoid issues it is best to use either, but not both.
WARNING: Due to limitations in metric names, topics with a period ('.') or underscore ('_') could collide. To avoid issues it is best to use either, but not both.
WARNING: Due to limitations in metric names, topics with a period ('.') or underscore ('_') could collide. To avoid issues it is best to use either, but not both.
Completed updating config for topic _schemas.

elasticsearch:
  enabled: true   # set this to false, if you want to provide your own ES instance.
  replicas: 3
  minimumMasterNodes: 1
  # Set replicas to 1 and uncomment this to allow the instance to be scheduled on
  # a master node when deploying on a single node Minikube / Kind / etc cluster.
  # antiAffinity: "soft"

  # # If your running a single replica cluster add the following helm value
  # clusterHealthCheckParams: "wait_for_status=yellow&timeout=1s"

  # # Shrink default JVM heap.
  # esJavaOpts: "-Xmx128m -Xms128m"

  # # Allocate smaller chunks of memory per pod.
  resources:
    requests:
      cpu: "0.1"
      memory: "100M"
    limits:
      cpu: "1000m"
      memory: "512M"

  # # Request smaller persistent volumes.
  volumeClaimTemplate:
    accessModes: ["ReadWriteOnce"]
    storageClassName: "rook-ceph-block"
    resources:
      requests:
        storage: 200M

0:02:11.189 [main] INFO  c.l.m.s.e.i.ESIndexBuilder:200 - Index datasetindex_v2_1662026531189 does not exist. Creating
java.net.SocketTimeoutException: 30,000 milliseconds timeout on connection http-outgoing-2 [ACTIVE]
	at org.elasticsearch.client.RestClient.extractAndWrapCause(RestClient.java:834)
	at org.elasticsearch.client.RestClient.performRequest(RestClient.java:259)
	at org.elasticsearch.client.RestClient.performRequest(RestClient.java:246)
	at org.elasticsearch.client.RestHighLevelClient.internalPerformRequest(RestHighLevelClient.java:1613)
	at org.elasticsearch.client.RestHighLevelClient.performRequest(RestHighLevelClient.java:1598)
	at org.elasticsearch.client.RestHighLevelClient.performRequestAndParseEntity(RestHighLevelClient.java:1565)
	at org.elasticsearch.client.IndicesClient.create(IndicesClient.java:145)
	at com.linkedin.metadata.search.elasticsearch.indexbuilder.ESIndexBuilder.createIndex(ESIndexBuilder.java:204)
	at com.linkedin.metadata.search.elasticsearch.indexbuilder.ESIndexBuilder.buildIndex(ESIndexBuilder.java:106)
	at com.linkedin.metadata.search.elasticsearch.indexbuilder.EntityIndexBuilder.buildIndex(EntityIndexBuilder.java:23)
	at com.linkedin.metadata.search.elasticsearch.indexbuilder.EntityIndexBuilders.buildAll(EntityIndexBuilders.java:21)
	at com.linkedin.metadata.search.elasticsearch.ElasticSearchService.configure(ElasticSearchService.java:37)
	at com.linkedin.metadata.kafka.hook.UpdateIndicesHook.<init>(UpdateIndicesHook.java:83)
	at sun.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method)
	at sun.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:62)
	at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45)
	at java.lang.reflect.Constructor.newInstance(Constructor.java:423)
	at org.springframework.beans.BeanUtils.instantiateClass(BeanUtils.java:211)
	at org.springframework.beans.factory.support.SimpleInstantiationStrategy.instantiate(SimpleInstantiationStrategy.java:117)
	at org.springframework.beans.factory.support.ConstructorResolver.instantiate(ConstructorResolver.java:311)
	at org.springframework.beans.factory.support.ConstructorResolver.autowireConstructor(ConstructorResolver.java:296)
	at org.springframework.beans.factory.support.AbstractAutowireCapableBeanFactory.autowireConstructor(AbstractAutowireCapableBeanFactory.java:1372)
	at org.springframework.beans.factory.support.AbstractAutowireCapableBeanFactory.createBeanInstance(AbstractAutowireCapableBeanFactory.java:1222)
	at org.springframework.beans.factory.support.AbstractAutowireCapableBeanFactory.doCreateBean(AbstractAutowireCapableBeanFactory.java:582)
	at org.springframework.beans.factory.support.AbstractAutowireCapableBeanFactory.createBean(AbstractAutowireCapableBeanFactory.java:542)
	at org.springframework.beans.factory.support.AbstractBeanFactory.lambda$doGetBean$0(AbstractBeanFactory.java:335)
	at org.springframework.beans.factory.support.DefaultSingletonBeanRegistry.getSingleton(DefaultSingletonBeanRegistry.java:234)
	at org.springframework.beans.factory.support.AbstractBeanFactory.doGetBean(AbstractBeanFactory.java:333)
	at org.springframework.beans.factory.support.AbstractBeanFactory.getBean(AbstractBeanFactory.java:208)
	at org.springframework.beans.factory.support.DefaultListableBeanFactory.preInstantiateSingletons(DefaultListableBeanFactory.java:953)
	at org.springframework.context.support.AbstractApplicationContext.finishBeanFactoryInitialization(AbstractApplicationContext.java:918)
	at org.springframework.context.support.AbstractApplicationContext.refresh(AbstractApplicationContext.java:583)
	at org.springframework.web.context.ContextLoader.configureAndRefreshWebApplicationContext(ContextLoader.java:401)
	at org.springframework.web.context.ContextLoader.initWebApplicationContext(ContextLoader.java:292)
	at org.springframework.web.context.ContextLoaderListener.contextInitialized(ContextLoaderListener.java:103)
	at org.eclipse.jetty.server.handler.ContextHandler.callContextInitialized(ContextHandler.java:1073)
	at org.eclipse.jetty.servlet.ServletContextHandler.callContextInitialized(ServletContextHandler.java:572)
	at org.eclipse.jetty.server.handler.ContextHandler.contextInitialized(ContextHandler.java:1002)
	at org.eclipse.jetty.servlet.ServletHandler.initialize(ServletHandler.java:746)
	at org.eclipse.jetty.servlet.ServletContextHandler.startContext(ServletContextHandler.java:379)
	at org.eclipse.jetty.webapp.WebAppContext.startWebapp(WebAppContext.java:1449)
	at org.eclipse.jetty.webapp.WebAppContext.startContext(WebAppContext.java:1414)
	at org.eclipse.jetty.server.handler.ContextHandler.doStart(ContextHandler.java:916)
	at org.eclipse.jetty.servlet.ServletContextHandler.doStart(ServletContextHandler.java:288)
	at org.eclipse.jetty.webapp.WebAppContext.doStart(WebAppContext.java:524)
	at org.eclipse.jetty.util.component.AbstractLifeCycle.start(AbstractLifeCycle.java:73)
	at org.eclipse.jetty.util.component.ContainerLifeCycle.start(ContainerLifeCycle.java:169)
	at org.eclipse.jetty.util.component.ContainerLifeCycle.doStart(ContainerLifeCycle.java:117)
	at org.eclipse.jetty.server.handler.AbstractHandler.doStart(AbstractHandler.java:97)
	at org.eclipse.jetty.util.component.AbstractLifeCycle.start(AbstractLifeCycle.java:73)
	at org.eclipse.jetty.util.component.ContainerLifeCycle.start(ContainerLifeCycle.java:169)
	at org.eclipse.jetty.util.component.ContainerLifeCycle.doStart(ContainerLifeCycle.java:117)
	at org.eclipse.jetty.server.handler.AbstractHandler.doStart(AbstractHandler.java:97)
	at org.eclipse.jetty.util.component.AbstractLifeCycle.start(AbstractLifeCycle.java:73)
	at org.eclipse.jetty.util.component.ContainerLifeCycle.start(ContainerLifeCycle.java:169)
	at org.eclipse.jetty.server.Server.start(Server.java:423)
	at org.eclipse.jetty.util.component.ContainerLifeCycle.doStart(ContainerLifeCycle.java:110)
	at org.eclipse.jetty.server.handler.AbstractHandler.doStart(AbstractHandler.java:97)
	at org.eclipse.jetty.server.Server.doStart(Server.java:387)
	at org.eclipse.jetty.util.component.AbstractLifeCycle.start(AbstractLifeCycle.java:73)
	at org.eclipse.jetty.runner.Runner.run(Runner.java:519)
	at org.eclipse.jetty.runner.Runner.main(Runner.java:564)
Caused by: java.net.SocketTimeoutException: 30,000 milliseconds timeout on connection http-outgoing-2 [ACTIVE]
	at org.apache.http.nio.protocol.HttpAsyncRequestExecutor.timeout(HttpAsyncRequestExecutor.java:387)
	at org.apache.http.impl.nio.client.InternalIODispatch.onTimeout(InternalIODispatch.java:92)
	at org.apache.http.impl.nio.client.InternalIODispatch.onTimeout(InternalIODispatch.java:39)
	at org.apache.http.impl.nio.reactor.AbstractIODispatch.timeout(AbstractIODispatch.java:175)
	at org.apache.http.impl.nio.reactor.BaseIOReactor.sessionTimedOut(BaseIOReactor.java:261)
	at org.apache.http.impl.nio.reactor.AbstractIOReactor.timeoutCheck(AbstractIOReactor.java:502)
	at org.apache.http.impl.nio.reactor.BaseIOReactor.validate(BaseIOReactor.java:211)
	at org.apache.http.impl.nio.reactor.AbstractIOReactor.execute(AbstractIOReactor.java:280)
	at org.apache.http.impl.nio.reactor.BaseIOReactor.execute(BaseIOReactor.java:104)
	at org.apache.http.impl.nio.reactor.AbstractMultiworkerIOReactor$Worker.run(AbstractMultiworkerIOReactor.java:591)
	at java.lang.Thread.run(Thread.java:748)
10:02:41.279 [main] INFO  c.l.m.s.e.i.EntityIndexBuilder:19 - Setting up index: chartindex_v2
10:02:41.284 [main] WARN  org.elasticsearch.client.RestClient:65 - request [HEAD <http://elasticsearch-master:9200/chartindex_v2?ignore_throttled=false&ignore_unavailable=false&expand_wildcards=open%2Cclosed&allow_no_indices=false>] returned 2 warnings: [299 Elasticsearch-7.16.2-2b937c44140b6559905130a8650c64dbd0879cfb "Elasticsearch built-in security features are not enabled. Without authentication, your cluster could be accessible to anyone. See <https://www.elastic.co/guide/en/elasticsearch/reference/7.16/security-minimal-setup.html> to enable security."],[299 Elasticsearch-7.16.2-2b937c44140b6559905130a8650c64dbd0879cfb "[ignore_throttled] parameter is deprecated because frozen indices have been deprecated. Consider cold or frozen tiers in place of frozen indices."]
10:02:41.285 [main] INFO  c.l.m.s.e.i.ESIndexBuilder:200 - Index chartindex_v2 does not exist. Creating