https://datahubproject.io logo
Join Slack
Powered by
Hi, I am trying to deploy a new datahub instance t...
# all-things-deployment
g
Hi, I am trying to deploy a new datahub instance through docker-compose. But datahub-gms is not working properly in log I found:
Copy code
+ exec dockerize -wait <http://elasticsearch:9200> -wait-http-header 'Accept: */*' -wait <tcp://mysql:3306> -wait <tcp://broker:29092> -timeout 240s java -Xms1g -Xmx1g -jar /jetty-runner.jar --jar jetty-util.jar --jar jetty-jmx.jar --config /datahub/datahub-gms/scripts/jetty.xml /datahub/datahub-gms/bin/war.war
2022/05/24 13:28:43 Waiting for: <http://elasticsearch:9200>
2022/05/24 13:28:43 Waiting for: <tcp://mysql:3306>
2022/05/24 13:28:43 Waiting for: <tcp://broker:29092>
2022/05/24 13:28:43 Connected to <tcp://broker:29092>
2022/05/24 13:28:43 Problem with dial: dial tcp: lookup mysql on 127.0.0.11:53: server misbehaving. Sleeping 1s
2022/05/24 13:28:43 Received 200 from <http://elasticsearch:9200>
2022/05/24 13:28:44 Problem with dial: dial tcp: lookup mysql on 127.0.0.11:53: server misbehaving. Sleeping 1s
.......
2022/05/24 13:28:57 Problem with dial: dial tcp: lookup mysql on 127.0.0.11:53: server misbehaving. Sleeping 1s

2022/05/24 13:29:11 Problem with dial: dial tcp 10.1.0.17:3306: connect: connection refused. Sleeping 1s
2022/05/24 13:29:12 Problem with dial: dial tcp 10.1.0.17:3306: connect: connection refused. Sleeping 1s
2022/05/24 13:29:13 Connected to <tcp://mysql:3306>
2022-05-24 13:29:13.793:INFO::main: Logging initialized @293ms to org.eclipse.jetty.util.log.StdErrLog
WARNING: jetty-runner is deprecated.
         See Jetty Documentation for startup options
         <https://www.eclipse.org/jetty/documentation/>
2022-05-24 13:29:13.823:INFO:oejr.Runner:main: Runner
2022-05-24 13:29:13.970:INFO:oeju.TypeUtil:main: JVM Runtime does not support Modules
2022-05-24 13:29:14.105:INFO:oejs.Server:main: jetty-9.4.20.v20190813; built: 2019-08-13T21:28:18.144Z; git: 84700530e645e812b336747464d6fbbf370c9a20; jvm 1.8.0_302-b08
2022-05-24 13:29:14.595:WARN:oejw.WebAppContext:main: Failed startup of context o.e.j.w.WebAppContext@50cbc42f{/,null,UNAVAILABLE}{file:///datahub/datahub-gms/bin/war.war}
java.util.zip.ZipException: invalid entry CRC (expected 0xd9b0e036 but got 0x978dbe6a)
        at java.util.zip.ZipInputStream.readEnd(ZipInputStream.java:394)
        at java.util.zip.ZipInputStream.read(ZipInputStream.java:196)
        at java.util.jar.JarInputStream.read(JarInputStream.java:207)
        at org.eclipse.jetty.util.IO.copy(IO.java:172)
        at org.eclipse.jetty.util.IO.copy(IO.java:122)
        at org.eclipse.jetty.util.resource.JarResource.copyTo(JarResource.java:218)
        at org.eclipse.jetty.webapp.WebInfConfiguration.unpack(WebInfConfiguration.java:636)
        at org.eclipse.jetty.webapp.WebInfConfiguration.preConfigure(WebInfConfiguration.java:140)
        at org.eclipse.jetty.webapp.WebAppContext.preConfigure(WebAppContext.java:488)
        at org.eclipse.jetty.webapp.WebAppContext.doStart(WebAppContext.java:523)
        at org.eclipse.jetty.util.component.AbstractLifeCycle.start(AbstractLifeCycle.java:72)
        at org.eclipse.jetty.util.component.ContainerLifeCycle.start(ContainerLifeCycle.java:169)
        at org.eclipse.jetty.util.component.ContainerLifeCycle.doStart(ContainerLifeCycle.java:117)
        at org.eclipse.jetty.server.handler.AbstractHandler.doStart(AbstractHandler.java:106)
        at org.eclipse.jetty.util.component.AbstractLifeCycle.start(AbstractLifeCycle.java:72)
        at org.eclipse.jetty.util.component.ContainerLifeCycle.start(ContainerLifeCycle.java:169)
        at org.eclipse.jetty.util.component.ContainerLifeCycle.doStart(ContainerLifeCycle.java:117)
        at org.eclipse.jetty.server.handler.AbstractHandler.doStart(AbstractHandler.java:106)
        at org.eclipse.jetty.util.component.AbstractLifeCycle.start(AbstractLifeCycle.java:72)
        at org.eclipse.jetty.util.component.ContainerLifeCycle.start(ContainerLifeCycle.java:169)
        at org.eclipse.jetty.server.Server.start(Server.java:407)
        at org.eclipse.jetty.util.component.ContainerLifeCycle.doStart(ContainerLifeCycle.java:110)
        at org.eclipse.jetty.server.handler.AbstractHandler.doStart(AbstractHandler.java:106)
        at org.eclipse.jetty.server.Server.doStart(Server.java:371)
        at org.eclipse.jetty.util.component.AbstractLifeCycle.start(AbstractLifeCycle.java:72)
        at org.eclipse.jetty.runner.Runner.run(Runner.java:520)
        at org.eclipse.jetty.runner.Runner.main(Runner.java:565)
I am using this compose file:
Copy code
networks:
  production-net:
    external: true
services:
  broker:
    container_name: broker
    depends_on:
    - zookeeper
    environment:
    - KAFKA_BROKER_ID=1
    - KAFKA_ZOOKEEPER_CONNECT=zookeeper:2181
    - KAFKA_LISTENER_SECURITY_PROTOCOL_MAP=PLAINTEXT:PLAINTEXT,PLAINTEXT_HOST:PLAINTEXT
    - KAFKA_ADVERTISED_LISTENERS=<PLAINTEXT://broker:29092>,PLAINTEXT_<HOST://localhost:9092>
    - KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR=1
    - KAFKA_GROUP_INITIAL_REBALANCE_DELAY_MS=0
    - KAFKA_HEAP_OPTS=-Xms256m -Xmx256m
    hostname: broker
    networks:
      production-net:
    image: confluentinc/cp-kafka:5.4.0
    ports:
    - 29092:29092
    - 9092:9092

  datahub-actions:
    depends_on:
    - datahub-gms
    environment:
    - GMS_HOST=datahub-gms
    - GMS_PORT=8080
    - KAFKA_BOOTSTRAP_SERVER=broker:29092
    - SCHEMA_REGISTRY_URL=<http://schema-registry:8081>
    - METADATA_AUDIT_EVENT_NAME=MetadataAuditEvent_v4
    - METADATA_CHANGE_LOG_VERSIONED_TOPIC_NAME=MetadataChangeLog_Versioned_v1
    - DATAHUB_SYSTEM_CLIENT_ID=__datahub_system
    - DATAHUB_SYSTEM_CLIENT_SECRET=JohnSnowKnowsNothing
    - KAFKA_PROPERTIES_SECURITY_PROTOCOL=PLAINTEXT
    hostname: datahub-actions
    networks:
      production-net:
    image: acryldata/acryl-datahub-actions:head
    restart: on-failure:5
    volumes:
    - ../hosts:/etc/hosts

  datahub-frontend-react:
    container_name: datahub-frontend-react
    depends_on:
    - datahub-gms
    environment:
    - DATAHUB_GMS_HOST=datahub-gms
    - DATAHUB_GMS_PORT=8080
    - DATAHUB_SECRET=YouKnowNothing
    - DATAHUB_APP_VERSION=1.0
    - DATAHUB_PLAY_MEM_BUFFER_SIZE=10MB
    - JAVA_OPTS=-Xms512m -Xmx512m -Dhttp.port=9002 -Dconfig.file=datahub-frontend/conf/application.conf
      -Djava.security.auth.login.config=datahub-frontend/conf/jaas.conf -Dlogback.configurationFile=datahub-frontend/conf/logback.xml
      -Dlogback.debug=false -Dpidfile.path=/dev/null
    - KAFKA_BOOTSTRAP_SERVER=broker:29092
    - DATAHUB_TRACKING_TOPIC=DataHubUsageEvent_v1
    - ELASTIC_CLIENT_HOST=elasticsearch
    - ELASTIC_CLIENT_PORT=9200
    hostname: datahub-frontend-react
    networks:
      production-net:
    image: linkedin/datahub-frontend-react:v0.8.34
    ports:
    - 9002:9002
    volumes:
    - ${HOME}/.datahub/plugins:/etc/datahub/plugins
    - ../hosts:/etc/hosts

  datahub-gms:
    container_name: datahub-gms
    depends_on:
    - mysql
    environment:
    - DATASET_ENABLE_SCSI=false
    - EBEAN_DATASOURCE_USERNAME=datahub
    - EBEAN_DATASOURCE_PASSWORD=datahub
    - EBEAN_DATASOURCE_HOST=mysql:3306
    - EBEAN_DATASOURCE_URL=jdbc:<mysql://mysql:3306/datahub?verifyServerCertificate=false&useSSL=true&useUnicode=yes&characterEncoding=UTF-8>
    - EBEAN_DATASOURCE_DRIVER=com.mysql.jdbc.Driver
    - KAFKA_BOOTSTRAP_SERVER=broker:29092
    - KAFKA_SCHEMAREGISTRY_URL=<http://schema-registry:8081>
    - ELASTICSEARCH_HOST=elasticsearch
    - ELASTICSEARCH_PORT=9200
    - GRAPH_SERVICE_IMPL=elasticsearch
    - JAVA_OPTS=-Xms1g -Xmx1g
    - ENTITY_REGISTRY_CONFIG_PATH=/datahub/datahub-gms/resources/entity-registry.yml
    - MAE_CONSUMER_ENABLED=true
    - MCE_CONSUMER_ENABLED=true
    - PE_CONSUMER_ENABLED=true
    - UI_INGESTION_ENABLED=true
    - UI_INGESTION_DEFAULT_CLI_VERSION=0.8.32.1
    hostname: datahub-gms
    networks:
      production-net:
    image: linkedin/datahub-gms:v0.8.34
    ports:
    - 9090:8080
    volumes:
    - ${HOME}/.datahub/plugins:/etc/datahub/plugins
    - ../hosts:/etc/hosts

  elasticsearch:
    container_name: elasticsearch
    environment:
    - discovery.type=single-node
    - xpack.security.enabled=false
    - ES_JAVA_OPTS=-Xms256m -Xmx256m -Dlog4j2.formatMsgNoLookups=true
    healthcheck:
      retries: 4
      start_period: 2m
      test:
      - CMD-SHELL
      - curl -sS --fail '<http://localhost:9200/_cluster/health?wait_for_status=yellow&timeout=0s>'
        || exit 1
    hostname: elasticsearch
    networks:
      production-net:
    image: elasticsearch:7.9.3
    mem_limit: 1g
    ports:
    - 9200:9200
    volumes:
    - esdata:/usr/share/elasticsearch/data
    - ../hosts:/etc/hosts

  elasticsearch-setup:
    container_name: elasticsearch-setup
    depends_on:
    - elasticsearch
    environment:
    - ELASTICSEARCH_HOST=elasticsearch
    - ELASTICSEARCH_PORT=9200
    - ELASTICSEARCH_PROTOCOL=http
    hostname: elasticsearch-setup
    networks:
      production-net:
    image: linkedin/datahub-elasticsearch-setup:v0.8.34

  kafka-setup:
    container_name: kafka-setup
    depends_on:
    - broker
    - schema-registry
    environment:
    - KAFKA_ZOOKEEPER_CONNECT=zookeeper:2181
    - KAFKA_BOOTSTRAP_SERVER=broker:29092
    hostname: kafka-setup
    networks:
      production-net:
    image: linkedin/datahub-kafka-setup:v0.8.34

  mysql:
    command: --character-set-server=utf8mb4 --collation-server=utf8mb4_bin
    container_name: mysql
    environment:
    - MYSQL_DATABASE=datahub
    - MYSQL_USER=datahub
    - MYSQL_PASSWORD=datahub
    - MYSQL_ROOT_PASSWORD=datahub
    hostname: mysql
    networks:
      production-net:
    image: mysql:5.7
    ports:
    - 3306:3306
    volumes:
    - ../mysql/init.sql:/docker-entrypoint-initdb.d/init.sql
    - mysqldata:/var/lib/mysql
    - ../hosts:/etc/hosts

  mysql-setup:
    container_name: mysql-setup
    depends_on:
    - mysql
    environment:
    - MYSQL_HOST=mysql
    - MYSQL_PORT=3306
    - MYSQL_USERNAME=datahub
    - MYSQL_PASSWORD=datahub
    - DATAHUB_DB_NAME=datahub
    hostname: mysql-setup
    networks:
      production-net:
    image: acryldata/datahub-mysql-setup:v0.8.34.1

  schema-registry:
    container_name: schema-registry
    depends_on:
    - zookeeper
    - broker
    environment:
    - SCHEMA_REGISTRY_HOST_NAME=schemaregistry
    - SCHEMA_REGISTRY_KAFKASTORE_CONNECTION_URL=zookeeper:2181
    hostname: schema-registry
    networks:
      production-net:
    image: confluentinc/cp-schema-registry:5.4.0
    ports:
    - 8081:8081

  zookeeper:
    container_name: zookeeper
    environment:
    - ZOOKEEPER_CLIENT_PORT=2181
    - ZOOKEEPER_TICK_TIME=2000
    hostname: zookeeper
    networks:
      production-net:
    image: confluentinc/cp-zookeeper:5.4.0
    ports:
    - 2181:2181
    volumes:
    - ./zkdata:/var/opt/zookeeper
    - ../hosts:/etc/hosts

version: '2.3'
volumes:
  esdata: null
  mysqldata: null
Have you ever seen this error?
Upgrading to version 0.8.35 the error seems disappeared
l
Hi @great-cpu-72376! Glad to hear the error as been resolved 🙂 Gentle reminder to please post large blocks of code/logs in slack message threads - it’s a HUGE help to ensure we are able to keep track of outstanding questions across our various support channels teamwork
8 Views
Open in Slack
PreviousNext
Powered by