readHiveTable(<databaseName>, <tableName>)

docker run -it --user root -v /my/path/recipes:/temp linkedin/datahub-ingestion:v0.8.41 ingest run --dry-run -c /temp/kafka-connect-to-datahub-kafka.yml

[2022-10-28 09:47:42,927] INFO     {datahub.cli.ingest_cli:99} - DataHub CLI version: 0.8.41+docker
[2022-10-28 09:47:42,952] INFO     {datahub.ingestion.run.pipeline:160} - Sink configured successfully. 
[2022-10-28 09:47:44,455] INFO     {datahub.ingestion.source.kafka_connect:866} - Connection to <address> is ok
[2022-10-28 09:47:44,456] ERROR    {datahub.ingestion.run.pipeline:126} - No JVM shared library file (libjvm.so) found. Try setting up the JAVA_HOME environment variable properly.
[2022-10-28 09:47:44,456] INFO     {datahub.cli.ingest_cli:115} - Starting metadata ingestion
[2022-10-28 09:47:44,456] INFO     {datahub.cli.ingest_cli:133} - Finished metadata pipeline

Failed to configure source (kafka-connect) due to No JVM shared library file (libjvm.so) found. Try setting up the JAVA_HOME environment variable properly.
No ~/.datahubenv file found, generating one for you...

"""dag name: social-insights"""
from datetime import timedelta, datetime
from airflow import DAG
from airflow.utils.dates import days_ago
from datahub.configuration.config_loader import load_config_file
from datahub.ingestion.run.pipeline import Pipeline

try:
    from airflow.operators.python import PythonOperator
except ModuleNotFoundError:
    from airflow.operators.python_operator import PythonOperator

default_args = {
    "owner": "airflow",
    "depends_on_past": False,
    "email": "<http://xxxx.com|xxxx.com>",
    "email_on_failure": False,
    "email_on_retry": False,
    "retries": 1,
    "retry_delay": timedelta(minutes=5),
    "execution_timeout": timedelta(minutes=120),
}


def template():
    """Run ingestion job: social-insights"""
    config = load_config_file("/social_insights.yaml")
    pipeline = Pipeline.create(config)
    pipeline.run()
    pipeline.raise_from_status()


with DAG(
    dag_id="social-insights",
    schedule_interval=timedelta(hours=1),
    start_date=days_ago(2),
) as dag:
    PythonOperator(
        task_id="social-insights",
        python_callable=template,
    )

. It will be skipped from lineage. The error was daemonic processes are not allowed to have children.

['Platform was not found in DataHub. Using postgres name as is'],\n"
           "              'metabase-dbname-2': ['Cannot determine database name for platform: postgres'],\n"
           "              'metabase-platform-3': ['Platform was not found in DataHub. Using postgres name as is'],\n"
           "              'metabase-dbname-3': ['Cannot determine database name for platform: postgres'],\n"
           "              'metabase-platform-1': ['Platform was not found in DataHub. Using h2 name as is'],\n"
           "              'metabase-dbname-1': ['Cannot determine database name for platform: h2']},\n"

~~~~ Execution Summary ~~~~

RUN_INGEST - {'errors': [],
 'exec_id': '2aa01dec-ff3a-4093-af01-9538d1fae92c',
 'infos': ['2022-10-30 18:30:00.193001 [exec_id=2aa01dec-ff3a-4093-af01-9538d1fae92c] INFO: Starting execution for task with name=RUN_INGEST',
           '2022-10-30 18:30:00.193303 [exec_id=2aa01dec-ff3a-4093-af01-9538d1fae92c] INFO: Caught exception EXECUTING '
           'task_id=2aa01dec-ff3a-4093-af01-9538d1fae92c, name=RUN_INGEST, stacktrace=Traceback (most recent call last):\n'
           '  File "/usr/local/lib/python3.10/site-packages/acryl/executor/execution/default_executor.py", line 113, in execute_task\n'
           '    task_event_loop = asyncio.new_event_loop()\n'
           '  File "/usr/local/lib/python3.10/asyncio/events.py", line 782, in new_event_loop\n'
           '    return get_event_loop_policy().new_event_loop()\n'
           '  File "/usr/local/lib/python3.10/asyncio/events.py", line 673, in new_event_loop\n'
           '    return self._loop_factory()\n'
           '  File "/usr/local/lib/python3.10/asyncio/unix_events.py", line 64, in __init__\n'
           '    super().__init__(selector)\n'
           '  File "/usr/local/lib/python3.10/asyncio/selector_events.py", line 53, in __init__\n'
           '    selector = selectors.DefaultSelector()\n'
           '  File "/usr/local/lib/python3.10/selectors.py", line 350, in __init__\n'
           '    self._selector = self._selector_cls()\n'
           'OSError: [Errno 24] Too many open files\n']}
Execution finished with errors.

%4|1667221196.244|FAIL|rdkafka#consumer-1| [thrd:<http://datahub-sbx2-frontend.amer-dev.XXXX.com:9092/bootstra|datahub-sbx2-frontend.amer-dev.XXXX.com:9092/bootstra>]: <http://datahub-sbx2-frontend.amer-dev.XXXX.com:9092/bootstrap|datahub-sbx2-frontend.amer-dev.XXXX.com:9092/bootstrap>: Connection setup timed out in state CONNECT (after 30090ms in state CONNECT)

profiling:
      enabled: true
      profile_table_level_only: true

Enable Profiling

profiling:
      enabled: true