Hi, Teams. I want to push the airflow task information to the datahub to create a lineage. https://datahubproject.io/docs/docker/airflow/local_airflow I followed all the guides in this document, and my DAG succeeded. However, pipelines are not visible in the datahub UI. What should I do? my dag is like:

from datetime import timedelta

from airflow import DAG
from airflow.operators.bash import BashOperator
from airflow.utils.dates import days_ago

from datahub_provider.entities import Dataset

default_args = {
    "owner": "airflow",
    "depends_on_past": False
}


with DAG(
    "datahub_lineage_backend_demo",
    default_args=default_args,
    description="An example DAG demonstrating the usage of DataHub's Airflow lineage backend.",
    schedule_interval=timedelta(days=1),
    start_date=days_ago(2),
    tags=["example_tag"],
    catchup=False,
) as dag:
    task1 = BashOperator(
        task_id="run_data_task",
        dag=dag,
        bash_command="echo 'This is where you might run your data tooling.'",
        inlets=[
            Dataset("snowflake", "mydb.schema.tableA"),
            Dataset("snowflake", "mydb.schema.tableB", "DEV")
        ],
        outlets=[Dataset("snowflake", "mydb.schema.tableD")],
    )

and also:

from datetime import timedelta
from airflow.operators.bash import BaseOperator, BashOperator

from airflow import DAG
from airflow.utils.dates import days_ago

import datahub.emitter.mce_builder as builder
from datahub_provider.operators.datahub import DatahubEmitterOperator

default_args = {
    "owner": "airflow",
    "depends_on_past": False,
    "retries": 1,
    "retry_delay": timedelta(minutes=5),
    "execution_timeout": timedelta(minutes=120),
}


with DAG(
        "datahub_lineage_emission_test",
        default_args=default_args,
        description="An example DAG demonstrating lineage emission within an Airflow DAG.",
        schedule_interval=timedelta(days=1),
        start_date=days_ago(2),
        catchup=False,
) as dag:
    # This example shows a SnowflakeOperator followed by a lineage emission. However, the
    # same DatahubEmitterOperator can be used to emit lineage in any context.

    lineage_dag_start = BashOperator(
        task_id="LINEAGE_START",
        dag=dag,
        bash_command="echo 'This is lineage test Start DAG'"
    )

    emit_lineage_task = DatahubEmitterOperator(
        task_id="emit_lineage",
        datahub_conn_id="datahub_rest_default",
        mces=[
            builder.make_lineage_mce(
                upstream_urns=[
                    builder.make_dataset_urn("s3", "mydb.schema.tableA"),
                    builder.make_dataset_urn("s3", "mydb.schema.tableB"),
                ],
                downstream_urn=builder.make_dataset_urn(
                    "s3", "mydb.schema.tableC"
                ),
            )
        ],
        dag=dag
    )

    lineage_finish = BashOperator(
        task_id="LINEAGE_FINISH",
        dag=dag,
        bash_command="echo 'This is lineage test Finish DAG'"
    )

    lineage_dag_start >> emit_lineage_task >> lineage_finish

Hi @melodic-ability-49840, do you think you could post the logs from the ingestion run here?

Do you mean Airflow DAG log? Then here you are. Or do I have to set the ingestion setting to draw Lineage? I didn’t see the information about ingestion in the guide.

Hi Kelly, let me check with the team and get back to you here- otherwise you can look under your ingestion tab and find logs by clicking into the specific runs listed there