https://datahubproject.io logo
Join Slack
Powered by
Hi guys! I need help deploying datahub using Docke...
# troubleshoot
h
Hi guys! I need help deploying datahub using Docker desktop with kubernetes enabled. I cloned the datahub-helm repo and after the front end is finally working I type the username and password ("datahub") but no success at login. I also tried modifying the 
values.xml
files for both datahub and prerequisites with no avail. I will post my 
values.xml
as a thread. I also added env values obtained from the datahub quickstart docker-compose files but doesn't seem to help either.
Copy code
# ------------------------
# prerequisites/values.xml
# ------------------------

# Default configuration for pre-requisites to get you started
# Copy this file and update to the configuration of choice
elasticsearch:
  enabled: true   # set this to false, if you want to provide your own ES instance.
  replicas: 3
  minimumMasterNodes: 1
  # Set replicas to 1 and uncomment this to allow the instance to be scheduled on
  # a master node when deploying on a single node Minikube / Kind / etc cluster.
  antiAffinity: "soft"

  # # If your running a single replica cluster add the following helm value
  clusterHealthCheckParams: "wait_for_status=yellow&timeout=1s"

# Official neo4j chart uses the Neo4j Enterprise Edition which requires a license
neo4j:
  enabled: false  # set this to true, if you have a license for the enterprise edition
  acceptLicenseAgreement: "yes"
  defaultDatabase: "graph.db"
  neo4jPassword: "datahub"
  # For better security, add password to neo4j-secrets k8s secret and uncomment below
  # existingPasswordSecret: neo4j-secrets
  core:
    standalone: true

# Deploys neo4j community version. Only supports single node
neo4j-community:
  enabled: true   # set this to false, if you have a license for the enterprise edition
  acceptLicenseAgreement: "yes"
  defaultDatabase: "graph.db"
  # For better security, add password to neo4j-secrets k8s secret and uncomment below
  existingPasswordSecret: neo4j-secrets

mysql:
  enabled: true
  auth:
    # For better security, add mysql-secrets k8s secret with mysql-root-password, mysql-replication-password and mysql-password
    existingSecret: mysql-secrets

postgresql:
  enabled: false
  auth:
    # For better security, add postgresql-secrets k8s secret with postgres-password, replication-password and password
    existingSecret: postgresql-secrets

cp-helm-charts:
  # Schema registry is under the community license
  cp-schema-registry:
    enabled: true
    kafka:
      bootstrapServers: "prerequisites-kafka:9092"  # <<release-name>>-kafka:9092
  cp-kafka:
    enabled: false
  cp-zookeeper:
    enabled: false
  cp-kafka-rest:
    enabled: false
  cp-kafka-connect:
    enabled: false
  cp-ksql-server:
    enabled: false
  cp-control-center:
    enabled: false

# Bitnami version of Kafka that deploys open source Kafka <https://artifacthub.io/packages/helm/bitnami/kafka>
kafka:
  enabled: true
Copy code
# ------------------
# datahub/values.xml
# ------------------

# Values to start up datahub after starting up the datahub-prerequisites chart with "prerequisites" release name
# Copy this chart and change configuration as needed.
datahub-gms:
  enabled: true
  image:
    repository: linkedin/datahub-gms
    tag: "v0.9.1"

datahub-frontend:
  enabled: true
  image:
    repository: linkedin/datahub-frontend-react
    tag: "v0.9.1"
  # Set up ingress to expose react front-end
  ingress:
    enabled: false
  extraEnvVars:
  - name: DATAHUB_GMS_HOST
    value: datahub-gms
  - name: DATAHUB_GMS_PORT
    value: 8080
  - name: DATAHUB_SECRET
    value: YouKnowNothing
  - name: DATAHUB_APP_VERSION
    value: 1.0
  - name: DATAHUB_PLAY_MEM_BUFFER_SIZE
    value: 10MB
  - name: JAVA_OPTS
    value: "-Xms512m -Xmx512m -Dhttp.port=9002 -Dconfig.file=datahub-frontend/conf/application.conf -Djava.security.auth.login.config=datahub-frontend/conf/jaas.conf -Dlogback.configurationFile=datahub-frontend/conf/logback.xml -Dlogback.debug=false -Dpidfile.path=/dev/null"
  - name: KAFKA_BOOTSTRAP_SERVER
    value: broker:29092
  - name: DATAHUB_TRACKING_TOPIC
    value: DataHubUsageEvent_v1
  - name: ELASTIC_CLIENT_HOST
    value: elasticsearch
  - name: ELASTIC_CLIENT_PORT
    value: 9200
  - name: METADATA_SERVICE_AUTH_ENABLED
    value: true

acryl-datahub-actions:
  enabled: true
  image:
    repository: acryldata/datahub-actions
    tag: "v0.0.7"
  resources:
    limits:
      memory: 512Mi
    requests:
      cpu: 300m
      memory: 256Mi
  extraEnvVars:
  - name: DATAHUB_GMS_HOST
    value: datahub-gms
  - name: DATAHUB_GMS_PORT
    value: 8080
  - name: KAFKA_BOOTSTRAP_SERVER
    value: broker:29092
  - name: SCHEMA_REGISTRY_URL
    value: <http://schema-registry:8081>
  - name: METADATA_AUDIT_EVENT_NAME
    value: MetadataAuditEvent_v4
  - name: METADATA_CHANGE_LOG_VERSIONED_TOPIC_NAME
    value: MetadataChangeLog_Versioned_v1
  - name: DATAHUB_SYSTEM_CLIENT_ID
    value: __datahub_system
  - name: DATAHUB_SYSTEM_CLIENT_SECRET
    value: JohnSnowKnowsNothing
  - name: KAFKA_PROPERTIES_SECURITY_PROTOCOL
    value: PLAINTEXT

datahub-mae-consumer:
  image:
    repository: linkedin/datahub-mae-consumer
    tag: "v0.9.1"

datahub-mce-consumer:
  image:
    repository: linkedin/datahub-mce-consumer
    tag: "v0.9.1"

datahub-ingestion-cron:
  enabled: false
  image:
    repository: acryldata/datahub-ingestion
    tag: "v0.9.1"

elasticsearchSetupJob:
  enabled: true
  image:
    repository: linkedin/datahub-elasticsearch-setup
    tag: "v0.9.1"
  podSecurityContext:
    fsGroup: 1000
  securityContext:
    runAsUser: 1000
  podAnnotations: {}

kafkaSetupJob:
  enabled: true
  image:
    repository: linkedin/datahub-kafka-setup
    tag: "v0.9.1"
  podSecurityContext:
    fsGroup: 1000
  securityContext:
    runAsUser: 1000
  podAnnotations: {}

mysqlSetupJob:
  enabled: true
  image:
    repository: acryldata/datahub-mysql-setup
    tag: "v0.9.1"
  podSecurityContext:
    fsGroup: 1000
  securityContext:
    runAsUser: 1000
  podAnnotations: {}

postgresqlSetupJob:
  enabled: false
  image:
    repository: acryldata/datahub-postgres-setup
    tag: "v0.9.1"
  podSecurityContext:
    fsGroup: 1000
  securityContext:
    runAsUser: 1000
  podAnnotations: {}

datahubUpgrade:
  enabled: true
  image:
    repository: acryldata/datahub-upgrade
    tag: "v0.9.1"
  batchSize: 1000
  batchDelayMs: 100
  noCodeDataMigration:
    sqlDbType: "MYSQL"
    # sqlDbType: "POSTGRES"
  podSecurityContext: {}
    # fsGroup: 1000
  securityContext: {}
    # runAsUser: 1000
  podAnnotations: {}
  restoreIndices:
    resources:
      limits:
        cpu: 500m
        memory: 512Mi
      requests:
        cpu: 300m
        memory: 256Mi

global:
  graph_service_impl: neo4j
  datahub_analytics_enabled: true
  datahub_standalone_consumers_enabled: false

  elasticsearch:
    host: "elasticsearch-master"
    port: "9200"
    skipcheck: "false"
    insecure: "false"
    extraEnvVars:
    - name: ELASTICSEARCH_VERSION
      value: 7.0

  kafka:
    bootstrap:
      server: "prerequisites-kafka:9092"
    zookeeper:
      server: "prerequisites-zookeeper:2181"
    schemaregistry:
      url: "<http://prerequisites-cp-schema-registry:8081>"
    extraEnvVars:
    - name: KAFKA_BROKER_ID
      value: 1
    - name: KAFKA_ZOOKEEPER_CONNECT
      value: zookeeper:2181
    - name: KAFKA_LISTENER_SECURITY_PROTOCOL_MAP
      value: PLAINTEXT:PLAINTEXT,PLAINTEXT_HOST:PLAINTEXT
    - name: KAFKA_ADVERTISED_LISTENERS
      value: <PLAINTEXT://broker:29092>,PLAINTEXT_<HOST://localhost:9092>
    - name: KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR
      value: 1
    - name: KAFKA_GROUP_INITIAL_REBALANCE_DELAY_MS
      value: 0
    - name: KAFKA_HEAP_OPTS
      value: -Xms256m -Xmx256m
    - name: KAFKA_CONFLUENT_SUPPORT_METRICS_ENABLE
      value: false

  neo4j:
    host: "prerequisites-neo4j-community:7474"
    uri: "<bolt://prerequisites-neo4j-community>"
    username: "neo4j"
    password:
      secretRef: neo4j-secrets
      secretKey: neo4j-password

  sql:
    datasource:
      host: "prerequisites-mysql:3306"
      hostForMysqlClient: "prerequisites-mysql"
      port: "3306"
      url: "jdbc:<mysql://prerequisites-mysql:3306/datahub?verifyServerCertificate=false&useSSL=true&useUnicode=yes&characterEncoding=UTF-8&enabledTLSProtocols=TLSv1.2>"
      driver: "com.mysql.cj.jdbc.Driver"
      username: "root"
      password:
        secretRef: mysql-secrets
        secretKey: mysql-root-password

  datahub:
    gms:
      port: "8080"
      nodePort: "30001"
      extraEnvVars:
      - name: DATAHUB_SERVER_TYPE
        value: ${DATAHUB_SERVER_TYPE:-quickstart}
      - name: DATAHUB_TELEMETRY_ENABLED
        value: ${DATAHUB_TELEMETRY_ENABLED:-true}
      - name: DATASET_ENABLE_SCSI
        value: false
      - name: EBEAN_DATASOURCE_USERNAME
        value: datahub
      - name: EBEAN_DATASOURCE_PASSWORD
        value: datahub
      - name: EBEAN_DATASOURCE_HOST
        value: mysql:3306
      - name: EBEAN_DATASOURCE_URL
        value: jdbc:<mysql://mysql:3306/datahub?verifyServerCertificate=false&useSSL=true&useUnicode=yes&characterEncoding=UTF-8>
      - name: EBEAN_DATASOURCE_DRIVER
        value: com.mysql.jdbc.Driver
      - name: KAFKA_BOOTSTRAP_SERVER
        value: broker:29092
      - name: KAFKA_SCHEMAREGISTRY_URL
        value: <http://schema-registry:8081>
      - name: ELASTICSEARCH_HOST
        value: elasticsearch
      - name: ELASTICSEARCH_PORT
        value: 9200
      - name: GRAPH_SERVICE_IMPL
        value: elasticsearch
      - name: JAVA_OPTS
        value: -Xms1g -Xmx1g
      - name: ENTITY_REGISTRY_CONFIG_PATH
        value: /datahub/datahub-gms/resources/entity-registry.yml
      - name: MAE_CONSUMER_ENABLED
        value: true
      - name: MCE_CONSUMER_ENABLED
        value: true
      - name: PE_CONSUMER_ENABLED
        value: true
      - name: UI_INGESTION_ENABLED
        value: true
      - name: ENTITY_SERVICE_ENABLE_RETENTION
        value: true
      - name: METADATA_SERVICE_AUTH_ENABLED
        value: true

    monitoring:
      enablePrometheus: true

    mae_consumer:
      port: "9091"
      nodePort: "30002"

    appVersion: "1.0"

    encryptionKey:
      secretRef: "datahub-encryption-secrets"
      secretKey: "encryption_key_secret"
      # Set to false if you'd like to provide your own secret.
      provisionSecret:
        enabled: true
        autoGenerate: true

    managed_ingestion:
      enabled: true
      defaultCliVersion: "0.9.1"

    metadata_service_authentication:
      enabled: false
      systemClientId: "__datahub_system"
      systemClientSecret:
        secretRef: "datahub-auth-secrets"
        secretKey: "token_service_signing_key"
      tokenService:
        signingKey:
          secretRef: "datahub-auth-secrets"
          secretKey: "token_service_signing_key"
        salt:
          secretRef: "datahub-auth-secrets"
          secretKey: "token_service_salt"
      # Set to false if you'd like to provide your own auth secrets
      provisionSecrets:
        enabled: true
        autoGenerate: true
On the front-end I am getting a 
connection refused
message from gms. And on gms I got a 
2022/11/03 13:50:29 Command exited with error: exit status 143
right after a 
terminated
signal was received and I was getting a lot of messages, while it was trying to run, similar to this one:
Copy code
Caused by: 
org.springframework.beans.TypeMismatchException: Failed to convert value of type 'java.lang.String' to required type 'int'; nested exception is java.lang.NumberFormatException: For input string: "<tcp://10.98.23.46:8080>"
kubectl get pods
prerequisites-cp-schema-registry-7cc6786995-7wg5j
only says 
PORT is deprecated. Please use SCHEMA_REGISTRY_LISTENERS instead.
and exited with status 
CrashLoopBackOff
.
34 Views
Open in Slack
PreviousNext
Powered by